rock_copy.s revision 2f0fcb93196badcdd803715656c809058d9f3114
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <sys/asm_linkage.h>
#include <sys/machthread.h>
#include <sys/privregs.h>
#if !defined(lint)
#include "assym.h"
#endif /* lint */
/*
* VIS_COPY_THRESHOLD indicates the minimum number of bytes needed
* to "break even" using FP/VIS-accelerated memory operations.
* The FPBLK code assumes a minimum number of bytes are available
* to be moved on entry. Check that code carefully before
* reducing VIS_COPY_THRESHOLD below 256.
*/
/*
* This shadows sys/machsystm.h which can't be included due to
* the lack of _ASM guards in include files it references.
* Change it here, change it there.
*/
#define VIS_COPY_THRESHOLD 256
/*
* TEST for very short copies
* Be aware that the maximum unroll for the short unaligned case
* is SHORTCOPY+1
*/
#define SHORTCOPY 3
#define CHKSIZE 39
/*
* Indicates that we're to trampoline to the error handler.
* Entry points bcopy, copyin_noerr, and copyout_noerr use this flag.
* kcopy, copyout, xcopyout, copyin, and xcopyin do not set this flag.
*/
#define FPUSED_FLAG 1
#define TRAMP_FLAG 2
#define MASK_FLAGS 3
/*
* LOFAULT_SET : Flag set by kzero and kcopy to indicate that t_lofault
* handler was set
*/
#define LOFAULT_SET 2
/*
* Number of outstanding prefetches.
* Testing with 1200 MHz Cheetah+ and Jaguar gives best results with
* two prefetches, one with a reach of 8*BLOCK_SIZE+8 and one with a
* reach of 5*BLOCK_SIZE. The double prefetch gives an typical improvement
* of 5% for large copies as compared to a single prefetch. The reason
* for the improvement is that with Cheetah and Jaguar, some prefetches
* are dropped due to the prefetch queue being full. The second prefetch
* reduces the number of cache lines that are dropped.
* Do not remove the double prefetch or change either FIRST_PREFETCH
* or SECOND_PREFETCH without extensive performance tests to prove
* there is no loss of performance.
* XXX: For ROCK, the prefetch depth can be upto 16, but sticking
* with 8 as of now pending more clarity on this.
*/
#define FIRST_PREFETCH 8
#define SECOND_PREFETCH 5
#define VIS_BLOCKSIZE 64
/*
* Size of stack frame in order to accomodate a 64-byte aligned
* floating-point register save area and 2 64-bit temp locations.
* All copy functions use two quadrants of fp registers; to assure a
* block-aligned two block buffer in which to save we must reserve
* three blocks on stack. Not all functions preserve %pfrs on stack
* or need to preserve %gsr but we use HWCOPYFRAMESIZE for all.
*
* _______________________________________ <-- %fp + STACK_BIAS
* | We may need to preserve 2 quadrants |
* | of fp regs, but since we do so with |
* | align to VIS_BLOCKSIZE bytes. So |
* | this area is 3 * VIS_BLOCKSIZE. | <-- - SAVED_FPREGS_OFFSET
* |-------------------------------------|
* | 8 bytes to save %fprs | <-- - SAVED_FPRS_OFFSET
* |-------------------------------------|
* | 8 bytes to save %gsr | <-- - SAVED_GSR_OFFSET
* ---------------------------------------
*/
#define ICACHE_LINE_SIZE 64
#define MEDIUM_MAX 255
#define PAGE_MASK 8191
#define ST_CACHE_ALIGN 127
#ifndef BSTORE_SIZE
#endif
/*
* Common macros used by the various versions of the block copy
* routines in this file.
*/
/*
* In FP copies if we do not have preserved data to restore over
* the fp regs we used then we must zero those regs to avoid
* exposing portions of the data to later threads (data security).
*
* Copy functions use either quadrants 1 and 3 or 2 and 4.
*
* FZEROQ3Q4: Zero quadrants 3 and 4, ie %d32 - %d46 and %d48 - %d62
*
*/
#define FZEROQ3Q4 \
/*
* Used to save and restore in-use fp registers when we want to use FP
* and find fp already in use and copy size still large enough to justify
* the additional overhead of this save and restore.
*
* A membar #Sync is needed before save to sync fp ops initiated before
* the call to the copy function (by whoever has fp in use); for example
* an earlier block load to the quadrant we are about to save may still be
* "in flight". A membar #Sync is required at the end of the save to
* sync our block store (the copy code is about to begin ldd's to the
* first quadrant). Note, however, that since Cheetah pipeline block load
* is blocking we can omit the initial membar before saving fp state (they're
* commented below in case of future porting to a chip that does not block
* on block load).
*
* Similarly: a membar #Sync before restore allows the block stores of
* the copy operation to complete before we fill the quadrants with their
* original data, and a membar #Sync after restore lets the block loads
* of the restore complete before we return to whoever has the fp regs
* in use. To avoid repeated membar #Sync we make it the responsibility
* of the copy code to membar #Sync immediately after copy is complete
* and before using the BLD_*_FROMSTACK macro.
*/
#if !defined(lint)
#define BST_FPQ3Q4_TOSTACK(tmp1) \
/* membar #Sync */ ;\
#define BLD_FPQ3Q4_FROMSTACK(tmp1) \
/* membar #Sync - provided at copy completion */ ;\
#endif
/*
* FP_NOMIGRATE and FP_ALLOWMIGRATE. Prevent migration (or, stronger,
* prevent preemption if there is no t_lwp to save FP state to on context
* switch) before commencing a FP copy, and reallow it on completion or
* in error trampoline paths when we were using FP copy.
*
* Both macros may call other functions, so be aware that all outputs are
* forfeit after using these macros. For this reason we do not pass registers
* to use - we just use any outputs we want.
*
* For fpRAS we need to perform the fpRAS mechanism test on the same
* CPU as we use for the copy operation, both so that we validate the
* CPU we perform the copy on and so that we know which CPU failed
* if a failure is detected. Hence we need to be bound to "our" CPU.
* This could be achieved through disabling preemption (and we have do it that
* way for threads with no t_lwp) but for larger copies this may hold
* higher priority threads off of cpu for too long (eg, realtime). So we
* make use of the lightweight t_nomigrate mechanism where we can (ie, when
* we have a t_lwp).
*
* Pseudo code:
*
* FP_NOMIGRATE:
*
* if (curthread->t_lwp) {
* thread_nomigrate();
* } else {
* kpreempt_disable();
* }
*
* FP_ALLOWMIGRATE:
*
* if (curthread->t_lwp) {
* thread_allowmigrate();
* } else {
* kpreempt_enable();
* }
*/
nop ;\
nop ;\
label1: ;\
nop ;\
nop ;\
label1: ;\
nop ;\
/*
* Copy a block of storage, returning an error code if `from' or
* `to' takes a kernel pagefault which cannot be resolved.
* Returns errno value on pagefault error, 0 if all ok
*/
#if defined(lint)
/* ARGSUSED */
int
{ return(0); }
#else /* lint */
.seg ".text"
.align 4
.kcopy_2:
.kcopy_4:
.kcopy_8:
/*
* We got here because of a fault during bcopy_more, called from kcopy or bcopy.
* Errno value is in %g1. bcopy_more uses fp quadrants 3 and 4.
*/
.copyerr:
4:
!
! and *not* to invoke any existing error handler. As far as
! bcopy is concerned, we only set t_lofault if there was an
!
1:
3:
!
! We're here via bcopy. There *must* have been an error handler
! in place otherwise we would have died a nasty death already.
!
jmp %l6 ! goto real handler
restore %g0, 0, %o0 ! dispose of copy window
/*
*/
.asciz "Unable to restore fp state after copy operation"
.align 4
.copyerr2:
/*
* We got here because of a fault during a small kcopy or bcopy.
* No floating point registers are used by the small copies.
* Errno value is in %g1.
*/
1:
3:
#endif /* lint */
/*
* Copy a block of storage - must not overlap (from + len <= to).
* Registers: l6 - saved t_lofault
* (for short copies, o4 - saved t_lofault)
*
* Copy a page of memory.
* Assumes double word alignment and a count >= 256.
*/
#if defined(lint)
/* ARGSUSED */
void
{}
#else /* lint */
.bcopy_2:
.bcopy_4:
.bcopy_8:
.align 16
.align 16
.align 16
.bc_med:
!
!
.align 16
!
!
.align 16
!
!
/*
* The _more entry points are not intended to be used directly by
* any caller from outside this file. They are provided to allow
* profiling and dtrace of the portions of the copy code that uses
* the floating point registers.
* This entry is particularly important as DTRACE (at least as of
* 4/2004) does not support leaf functions.
*/
!
! We've already captured whether t_lofault was zero on entry.
! We need to mark ourselves as being from bcopy since both
! kcopy and bcopy use the same code path. If TRAMP_FLAG is set
! returning.
!
/*
* Copies that reach here are larger than VIS_COPY_THRESHOLD bytes
* Also, use of FP registers has been tested to be enabled
*/
.do_copy:
.align 16
.medium:
.med1:
.med2:
/*
* Handle all cases where src and dest are aligned on word
* or long word boundaries. Use unrolled loops for better
* performance. This option wins over standard large data
* move when source and destination is in cache for medium
* to short data moves.
*/
/*
* no need to put prefetch in loop as prefetches have
* already been issued for maximum loop size
*/
.medw16:
.medw15:
nop !
nop !
/*
* Special case for handling when src and dest are both long word aligned
* and total data to move is between SMALL_MAX and MED_MAX bytes
*/
.align 16
/*
* no need to put prefetch in loop as prefetches have
* already been issued for maximum loop size
*/
.medl32:
.medl31:
nop !
.medl15:
.align 16
! boundary in REALSRC compared to in DST
!
! Examples: Let # denote bytes that should not be accessed
! Let x denote a byte already copied to align DST
! Let . and - denote bytes not yet copied
! Let | denote double alignment boundaries
!
! DST: ######xx|........|--------|..###### CNT = 18
! i1
!
! SRC = -3: REALSRC: ###xx...|.....---|-----..#|######## TMP = 8
! i0
!
! SRC = 0: REALSRC: ######xx|........|--------|..###### TMP = 16-8 = 8
! i0
!
! SRC = +1: REALSRC: #######x|x.......|.-------|-..##### TMP = 16-8 = 8
! i0
wr %g0, ASI_CACHE_SPARING_P, %asi
or %g0, -8, TMP
alignaddr REALSRC, %g0, REALSRC ! set GSR.ALIGN and align REALSRC
movrlz SRC, %g0, TMP ! subtract 8 from i2+i3 only if i3>=0
add TMP, CNT, TMP
add TMP, SRC, TMP
bleu %ncc, .med4
andn TMP, 7, TMP ! 8 byte aligned count
and TMP, VIS_BLOCKSIZE-1, TMP ! bytes till DST block aligned
.med4:
brgez,a SRC, .beginmedloop
ldda [REALSRC-8]%asi, %d32
add REALSRC, SRC, REALSRC ! back up REALSRC
.med5:
ldda [REALSRC]ASI_FL8_P, %d34
inc REALSRC
andcc REALSRC, 7, %g0
bnz %ncc, .med5
.medloop:
.medloop1:
.medloop2:
.medpst1:
.medpst2:
subcc CNT, 8, CNT
std %d32, [DST]
bz %ncc, .mediumexit
add DST, 8, DST
.medpst3:
ldub [REALSRC], SRC
deccc CNT
inc REALSRC
stb SRC, [DST]
bgu %ncc, .medpst3
inc DST
.mediumexit:
ba,pt %ncc, .bcb_exit
nop
.align ICACHE_LINE_SIZE
.large:
! The following test for BSTORE_SIZE is used to decide whether
! to store data with a block store or with individual stores.
! The block store wins when the amount of data is so large
! that it is causes other application data to be moved out
! of the L1 or L2 cache.
! On a Panther, block store can lose more often because block
! store forces the stored data to be removed from the L3 cache.
!
sethi %hi(BSTORE_SIZE), TMP
or TMP, %lo(BSTORE_SIZE), TMP
cmp CNT, TMP
bgu %ncc, .xlarge
! DST I/O Destination is 64-byte aligned
.large1:
ldda [REALSRC]%asi, %d34
faligndata %d44, %d46, %d60
ldda [REALSRC + 0x08]%asi, %d36
faligndata %d46, %d32, %d62
std %d48, [DST]
std %d50, [DST+8]
std %d52, [DST+16]
std %d54, [DST+24]
std %d56, [DST+32]
std %d58, [DST+40]
std %d60, [DST+48]
std %d62, [DST+56]
sub CNT, VIS_BLOCKSIZE, CNT ! update count
prefetch [DST + (6 * VIS_BLOCKSIZE)], #n_writes
prefetch [DST + (3 * VIS_BLOCKSIZE)], #n_writes
add DST, VIS_BLOCKSIZE, DST ! update DST
ldda [REALSRC + 0x10]%asi, %d38
faligndata %d32, %d34, %d48
ldda [REALSRC + 0x18]%asi, %d40
faligndata %d34, %d36, %d50
ldda [REALSRC + 0x20]%asi, %d42
faligndata %d36, %d38, %d52
ldda [REALSRC + 0x28]%asi, %d44
faligndata %d38, %d40, %d54
ldda [REALSRC + 0x30]%asi, %d46
faligndata %d40, %d42, %d56
ldda [REALSRC + 0x38]%asi, %d32
faligndata %d42, %d44, %d58
cmp CNT, VIS_BLOCKSIZE + 8
prefetch [REALSRC + (5 * VIS_BLOCKSIZE)], #one_read
bgu,pt %ncc, .large1
add REALSRC, VIS_BLOCKSIZE, REALSRC
faligndata %d44, %d46, %d60
faligndata %d46, %d32, %d62
stda %d48, [DST]ASI_BLK_P ! store 64 bytes, bypass cache
cmp CNT, VIS_BLOCKSIZE
bne %ncc, .large2 ! exactly 1 blk remaining?
add DST, VIS_BLOCKSIZE, DST ! update DST
brz,a SRC, .large3 ! is SRC double aligned ?
ldd [REALSRC], %d34
.large2:
add TMP, CNT, TMP ! TMP was already set to 0 or -8
add TMP, SRC, TMP
ba .beginmedloop
andn TMP, 7, TMP
.large3:
ldd [REALSRC + 0x08], %d36
ldd [REALSRC + 0x10], %d38
ldd [REALSRC + 0x18], %d40
ldd [REALSRC + 0x20], %d42
ldd [REALSRC + 0x28], %d44
ldd [REALSRC + 0x30], %d46
.align 16
.xlarge:
! %d32 I/O Already loaded with Source data from [REALSRC-8]
! CNT I/O Count (number of bytes that need to be written)
! SRC I Not written. If zero, then REALSRC is double aligned.
! TMP O The number of doubles that remain to be written.
! Load the rest of the current block
! Recall that REALSRC is further into source buffer
! than DST is into the destination buffer.
! prefetch [REALSRC + (3 * VIS_BLOCKSIZE)], #one_read
! executed in delay slot for branch to .xlarge
prefetch [REALSRC + (4 * VIS_BLOCKSIZE)], #one_read
prefetch [REALSRC + (5 * VIS_BLOCKSIZE)], #one_read
ldda [REALSRC]%asi, %d34
prefetch [REALSRC + (6 * VIS_BLOCKSIZE)], #one_read
ldda [REALSRC + 0x8]%asi, %d36
faligndata %d32, %d34, %d48
ldda [REALSRC + 0x10]%asi, %d38
faligndata %d34, %d36, %d50
ldda [REALSRC + 0x18]%asi, %d40
faligndata %d36, %d38, %d52
ldda [REALSRC + 0x20]%asi, %d42
or %g0, -8, TMP ! if SRC >= 0, TMP = -8
faligndata %d38, %d40, %d54
ldda [REALSRC + 0x28]%asi, %d44
movrlz SRC, %g0, TMP ! if SRC < 0, TMP = 0 (needed later)
faligndata %d40, %d42, %d56
ldda [REALSRC + 0x30]%asi, %d46
faligndata %d42, %d44, %d58
ldda [REALSRC + 0x38]%asi, %d32
sub CNT, VIS_BLOCKSIZE, CNT ! update count
prefetch [REALSRC + (7 * VIS_BLOCKSIZE)], #one_read
add REALSRC, VIS_BLOCKSIZE, REALSRC ! update REALSRC
! This point is 32-byte aligned since 24 instructions appear since
! the previous alignment directive.
! Main loop. Write previous block. Load rest of current block.
.xlarge1:
.xlarge2:
.xlarge3:
! Don't need to load [REALSRC + 0x38] since we were
! already 8 bytes ahead in REALSRC to start with.
stda %d32, [DST]ASI_BLK_P
.bcb_exit:
membar #Sync
ldx [%fp + STACK_BIAS - SAVED_GSR_OFFSET], %o2 ! restore gsr
wr %o2, 0, %gsr
ld [%fp + STACK_BIAS - SAVED_FPRS_OFFSET], %o3
! No need to save regs if either FEF or DU are clear
and %o3, FPRS_FEF|FPRS_DU, %o2
xorcc %o2, FPRS_FEF|FPRS_DU, %g0
bnz,pt %icc, 4f
nop
BLD_FPQ3Q4_FROMSTACK(%o2)
ba,pt %ncc, 5f
wr %o3, 0, %fprs ! restore fprs
4:
FZEROQ3Q4
wr %o3, 0, %fprs ! restore fprs
5:
mov %g5, %o0 ! copy dest address
call rock_sync_icache
mov %g2, %o1 ! saved size
membar #Sync ! sync error barrier
andn %l6, MASK_FLAGS, %l6
stn %l6, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
FP_ALLOWMIGRATE(5, 6)
ret
restore %g0, 0, %o0
SET_SIZE(bcopy_more)
#endif /* lint */
/*
* Block copy with possibly overlapped operands.
*/
#if defined(lint)
/*ARGSUSED*/
void
ovbcopy(const void *from, void *to, size_t count)
{}
#else /* lint */
ENTRY(ovbcopy)
tst %o2 ! check count
bgu,a %ncc, 1f ! nothing to do or bad arguments
subcc %o0, %o1, %o3 ! difference of from and to address
retl ! return
nop
1:
bneg,a %ncc, 2f
neg %o3 ! if < 0, make it positive
2: cmp %o2, %o3 ! cmp size and abs(from - to)
bleu %ncc, bcopy ! if size <= abs(diff): use bcopy,
.empty ! no overlap
cmp %o0, %o1 ! compare from and to addresses
blu %ncc, .ov_bkwd ! if from < to, copy backwards
nop
!
! Copy forwards.
!
.ov_fwd:
ldub [%o0], %o3 ! read from address
inc %o0 ! inc from address
stb %o3, [%o1] ! write to address
deccc %o2 ! dec count
bgu %ncc, .ov_fwd ! loop till done
inc %o1 ! inc to address
retl ! return
nop
!
! Copy backwards.
!
.ov_bkwd:
deccc %o2 ! dec count
ldub [%o0 + %o2], %o3 ! get byte at end of src
bgu %ncc, .ov_bkwd ! loop till done
stb %o3, [%o1 + %o2] ! delay slot, store at end of dst
retl ! return
nop
SET_SIZE(ovbcopy)
#endif /* lint */
/*
* hwblkpagecopy()
*
* Copies exactly one page. This routine assumes the caller (ppcopy)
* has already disabled kernel preemption and has checked
* use_hw_bcopy. Preventing preemption also prevents cpu migration.
*/
#ifdef lint
/*ARGSUSED*/
void
hwblkpagecopy(const void *src, void *dst)
{ }
#else /* lint */
ENTRY(hwblkpagecopy)
save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
! %i0 - source address (arg)
! %i1 - destination address (arg)
! %i2 - length of region (not arg)
! %l0 - saved fprs
! %l1 - pointer to saved fpregs
rd %fprs, %l0 ! check for unused fp
! FPU enabled ? If not, enable it.
btst FPRS_FEF, %l0
bz,a,pt %icc, 1f
wr %g0, FPRS_FEF, %fprs
! FPU enabled, but is Q3Q4 dirty ? If yes, save them.
btst FPRS_DU, %l0
bz,pn %icc, 1f
nop
BST_FPQ3Q4_TOSTACK(%l1)
1: set PAGESIZE, CNT
mov %i1, %o0 ! store destination address for flushing
mov REALSRC, SRC
prefetch [SRC], #one_read
prefetch [SRC + (1 * VIS_BLOCKSIZE)], #one_read
prefetch [SRC + (2 * VIS_BLOCKSIZE)], #one_read
prefetch [SRC + (3 * VIS_BLOCKSIZE)], #one_read
ldd [SRC], %d32
#if FIRST_PREFETCH > 4
prefetch [SRC + (4 * VIS_BLOCKSIZE)], #one_read
#endif
ldd [SRC + 0x08], %d34
#if FIRST_PREFETCH > 5
prefetch [SRC + (5 * VIS_BLOCKSIZE)], #one_read
#endif
ldd [SRC + 0x10], %d36
#if FIRST_PREFETCH > 6
prefetch [SRC + (6 * VIS_BLOCKSIZE)], #one_read
#endif
faligndata %d32, %d34, %d48
ldd [SRC + 0x18], %d38
#if FIRST_PREFETCH > 7
prefetch [SRC + (7 * VIS_BLOCKSIZE)], #one_read
#endif
faligndata %d34, %d36, %d50
ldd [SRC + 0x20], %d40
faligndata %d36, %d38, %d52
ldd [SRC + 0x28], %d42
faligndata %d38, %d40, %d54
ldd [SRC + 0x30], %d44
faligndata %d40, %d42, %d56
ldd [SRC + 0x38], %d46
faligndata %d42, %d44, %d58
ldd [SRC + VIS_BLOCKSIZE], %d32
sub CNT, VIS_BLOCKSIZE, CNT
add SRC, VIS_BLOCKSIZE, SRC
ba,a,pt %ncc, 2f
nop
.align ICACHE_LINE_SIZE
2:
ldd [SRC + 0x08], %d34
faligndata %d44, %d46, %d60
ldd [SRC + 0x10], %d36
faligndata %d46, %d32, %d62
stda %d48, [DST]ASI_BLK_P
ldd [SRC + 0x18], %d38
faligndata %d32, %d34, %d48
ldd [SRC + 0x20], %d40
faligndata %d34, %d36, %d50
ldd [SRC + 0x28], %d42
faligndata %d36, %d38, %d52
ldd [SRC + 0x30], %d44
faligndata %d38, %d40, %d54
ldd [SRC + 0x38], %d46
faligndata %d40, %d42, %d56
ldd [SRC + VIS_BLOCKSIZE], %d32
faligndata %d42, %d44, %d58
prefetch [SRC + ((FIRST_PREFETCH) * VIS_BLOCKSIZE) + 8], #one_read
sub CNT, VIS_BLOCKSIZE, CNT
add DST, VIS_BLOCKSIZE, DST
cmp CNT, VIS_BLOCKSIZE + 8
prefetch [SRC + ((SECOND_PREFETCH) * VIS_BLOCKSIZE)], #one_read
bgu,pt %ncc, 2b
add SRC, VIS_BLOCKSIZE, SRC
! trailing block
ldd [SRC + 0x08], %d34
faligndata %d44, %d46, %d60
ldd [SRC + 0x10], %d36
faligndata %d46, %d32, %d62
stda %d48, [DST]ASI_BLK_P
ldd [SRC + 0x18], %d38
ldd [SRC + 0x20], %d40
ldd [SRC + 0x28], %d42
ldd [SRC + 0x30], %d44
ldd [SRC + 0x38], %d46
sub CNT, VIS_BLOCKSIZE, CNT
add DST, VIS_BLOCKSIZE, DST
add SRC, VIS_BLOCKSIZE, SRC
stda %d32, [DST]ASI_BLK_P
set PAGESIZE, %o1
call rock_sync_icache
nop
membar #Sync
btst FPRS_DU, %l0
bz,pt %icc, 2f
nop
BLD_FPQ3Q4_FROMSTACK(%l3)
ba 3f
nop
2: FZEROQ3Q4
3: wr %l0, 0, %fprs ! restore fprs
ret
restore %g0, 0, %o0
SET_SIZE(hwblkpagecopy)
#endif /* lint */
/*
* Transfer data to and from user space -
* Note that these routines can cause faults
* It is assumed that the kernel has nothing at
* less than KERNELBASE in the virtual address space.
*
* Note that copyin(9F) and copyout(9F) are part of the
*
* Sigh.
*
* allows other callers (e.g. uiomove(9F)) to work correctly.
* Given that these are used pretty heavily, we expand the calling
* sequences inline for all flavours (rather than making wrappers).
*
* There are also stub routines for xcopyout_little and xcopyin_little,
* which currently are intended to handle requests of <= 16 bytes from
* do_unaligned. Future enhancement to make them handle 8k pages efficiently
* is left as an exercise...
*/
/*
* Copy user data to kernel space (copyOP/xcopyOP/copyOP_noerr)
*
* General theory of operation:
*
* The only difference between copy{in,out} and
* xcopy{in,out} is in the error handling routine they invoke
* when a memory access error occurs. xcopyOP returns the errno
* while copyOP returns -1 (see above). copy{in,out}_noerr set
* a special flag (by oring the TRAMP_FLAG into the fault handler address)
* if they are called with a fault handler already in place. That flag
* causes the default handlers to trampoline to the previous handler
* upon an error.
*
*
*
*
*/
/*
* Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
*/
#if defined(lint)
#else /* lint */
/*
* We save the arguments in the following registers in case of a fault:
* kaddr - %l1
* uaddr - %l2
* count - %l3
*/
#define SAVE_COUNT %l3
#define SM_SAVE_SRC %g4
#define SM_SAVE_DST %g5
#define SM_SAVE_COUNT %o5
#define REAL_LOFAULT %l4
/*
* Generic copyio fault handler. This is the first line of defense when a
* fault occurs in (x)copyin/(x)copyout. In order for this to function
* properly, the value of the 'real' lofault handler should be in REAL_LOFAULT.
* This allows us to share common code for all the flavors of the copy
* operations, including the _noerr versions.
*
* Note that this function will restore the original input parameters before
* calling REAL_LOFAULT. So the real handler can vector to the appropriate
* member of the t_copyop structure, if needed.
*/
4:
1:
#endif
#if defined(lint)
/*ARGSUSED*/
int
{ return (0); }
#else /* lint */
.align 16
.align 16
.align 16
.align 16
.co_med:
!
!
.align 16
!
!
.align 16
!
!
/*
* We got here because of a fault during short copyout.
*/
3:
/*
* The _more entry points are not intended to be used directly by
* any caller from outside this file. They are provided to allow
* profiling and dtrace of the portions of the copy code that uses
* the floating point registers.
* This entry is particularly important as DTRACE (at least as of
* 4/2004) does not support leaf functions.
*/
/*
* Copy outs that reach here are larger than VIS_COPY_THRESHOLD bytes
*/
2:
#if FIRST_PREFETCH > 4
#endif
#if FIRST_PREFETCH > 5
#endif
#if FIRST_PREFETCH > 6
#endif
#if FIRST_PREFETCH > 7
#endif
1:
3:
2:
4:
4:
1:
/*
* We got here because of a fault during copyout.
*/
2:
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
{ return (0); }
#else /* lint */
/*
* We got here because of fault during xcopyout
* Errno value is in ERRNO
*/
2:
3:
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
{ return (0); }
#else /* lint */
2:
#endif /* lint */
/*
* Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
*/
#if defined(lint)
/*ARGSUSED*/
int
{ return (0); }
#else /* lint */
.copyin_2:
.copyin_4:
.copyin_8:
.align 16
.align 16
.align 16
.align 16
.ci_med:
!
!
.align 16
!
!
.align 16
!
!
3:
/*
* The _more entry points are not intended to be used directly by
* any caller from outside this file. They are provided to allow
* profiling and dtrace of the portions of the copy code that uses
* the floating point registers.
* This entry is particularly important as DTRACE (at least as of
* 4/2004) does not support leaf functions.
*/
/*
* Copy ins that reach here are larger than VIS_COPY_THRESHOLD bytes
*/
2:
#if FIRST_PREFETCH > 4
#endif
#if FIRST_PREFETCH > 5
#endif
#if FIRST_PREFETCH > 6
#endif
#if FIRST_PREFETCH > 7
#endif
1:
3:
2:
4:
4:
1:
/*
* We got here because of a fault during copyin
*/
2:
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
{ return (0); }
#else /* lint */
/*
* We got here because of fault during xcopyin
* Errno value is in ERRNO
*/
2:
3:
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
{ return (0); }
#else /* lint */
2:
#endif /* lint */
/*
* Copy a block of storage - must not overlap (from + len <= to).
* No fault handler installed (to be called under on_fault())
*/
#if defined(lint)
/* ARGSUSED */
void
{}
#else /* lint */
#endif /* lint */
/*
* Copy a block of storage - must not overlap (from + len <= to).
* No fault handler installed (to be called under on_fault())
*/
#if defined(lint)
/* ARGSUSED */
void
{}
#else /* lint */
#endif /* lint */
/*
* hwblkclr - clears block-aligned, block-multiple-sized regions that are
* longer than 256 bytes in length using spitfire's block stores. If
* the criteria for using this routine are not met then it calls bzero
* and returns 1. Otherwise 0 is returned indicating success.
* Caller is responsible for ensuring use_hw_bzero is true and that
* kpreempt_disable() has been called.
*/
#ifdef lint
/*ARGSUSED*/
int
{
return(0);
}
#else /* lint */
#ifdef ROCK_CR_6654578
#endif
#ifdef ROCK_CR_6654578
#endif
#ifdef ROCK_CR_6654578
#endif
#ifdef ROCK_CR_6654578
#endif
.pz_zinst:
#ifdef ROCK_CR_6654578
#endif
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
void
{}
#else /*!lint */
/*
* Copy 32 bytes of data from src (%o0) to dst (%o1)
* using physical addresses.
*/
#endif /* lint */
/*
* Zero a block of storage.
*
* uzero is used by the kernel to zero a block in user address space.
*/
#if defined(lint)
/* ARGSUSED */
int
{ return(0); }
/* ARGSUSED */
void
{}
#else /* lint */
!
!
!
!
/*
* We got here because of a fault during kzero or if
* uzero or bzero was called with t_lofault non-zero.
* Otherwise we've already run screaming from the room.
* Errno value is in %g1. Note that we're here iff
* we did set t_lofault.
*/
.zeroerr:
!
!
!
!
1:
2:
!
!
retl ! return
3:
!
! We're here because %o5 was non-zero. It was non-zero
! because either LOFAULT_SET was present, a previous fault
! handler was present or both. In all cases we need to reset
! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
! before we either simply return the error or we invoke the
! previously specified handler.
!
be %ncc, 2b
stn %o5, [THREAD_REG + T_LOFAULT]
jmp %o5 ! goto real handler
nop
SET_SIZE(kzero)
SET_SIZE(uzero)
#endif /* lint */
/*
* Zero a block of storage.
*/
#if defined(lint)
/* ARGSUSED */
void
bzero(void *addr, size_t count)
{}
#else /* lint */
ENTRY(bzero)
wr %g0, ASI_P, %asi
ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector
tst %o5
bz,pt %ncc, .do_zero
sethi %hi(.zeroerr), %o2
or %o2, %lo(.zeroerr), %o2
membar #Sync ! sync error barrier
stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector
.do_zero:
cmp %o1, 7
blu,pn %ncc, .byteclr
nop
cmp %o1, 15
blu,pn %ncc, .wdalign
nop
andcc %o0, 7, %o3 ! is add aligned on a 8 byte bound
bz,pt %ncc, .blkalign ! already double aligned
sub %o3, 8, %o3 ! -(bytes till double aligned)
add %o1, %o3, %o1 ! update o1 with new count
1:
stba %g0, [%o0]%asi
inccc %o3
bl,pt %ncc, 1b
inc %o0
! Now address is double aligned
.blkalign:
cmp %o1, 0x80 ! check if there are 128 bytes to set
blu,pn %ncc, .bzero_small
mov %o1, %o3
andcc %o0, 0x3f, %o3 ! is block aligned?
bz,pt %ncc, .bzero_blk
sub %o3, 0x40, %o3 ! -(bytes till block aligned)
add %o1, %o3, %o1 ! o1 is the remainder
! Clear -(%o3) bytes till block aligned
1:
stxa %g0, [%o0]%asi
addcc %o3, 8, %o3
bl,pt %ncc, 1b
add %o0, 8, %o0
.bzero_blk:
and %o1, 0x3f, %o3 ! calc bytes left after blk clear
andn %o1, 0x3f, %o4 ! calc size of blocks in bytes
cmp %o4, 0x100 ! 256 bytes or more
blu,pn %ncc, 3f
nop
2:
stxa %g0, [%o0+0x0]%asi
stxa %g0, [%o0+0x40]%asi
stxa %g0, [%o0+0x80]%asi
stxa %g0, [%o0+0xc0]%asi
stxa %g0, [%o0+0x8]%asi
stxa %g0, [%o0+0x10]%asi
stxa %g0, [%o0+0x18]%asi
stxa %g0, [%o0+0x20]%asi
stxa %g0, [%o0+0x28]%asi
stxa %g0, [%o0+0x30]%asi
stxa %g0, [%o0+0x38]%asi
stxa %g0, [%o0+0x48]%asi
stxa %g0, [%o0+0x50]%asi
stxa %g0, [%o0+0x58]%asi
stxa %g0, [%o0+0x60]%asi
stxa %g0, [%o0+0x68]%asi
stxa %g0, [%o0+0x70]%asi
stxa %g0, [%o0+0x78]%asi
stxa %g0, [%o0+0x88]%asi
stxa %g0, [%o0+0x90]%asi
stxa %g0, [%o0+0x98]%asi
stxa %g0, [%o0+0xa0]%asi
stxa %g0, [%o0+0xa8]%asi
stxa %g0, [%o0+0xb0]%asi
stxa %g0, [%o0+0xb8]%asi
stxa %g0, [%o0+0xc8]%asi
stxa %g0, [%o0+0xd0]%asi
stxa %g0, [%o0+0xd8]%asi
stxa %g0, [%o0+0xe0]%asi
stxa %g0, [%o0+0xe8]%asi
stxa %g0, [%o0+0xf0]%asi
stxa %g0, [%o0+0xf8]%asi
sub %o4, 0x100, %o4
cmp %o4, 0x100
bgu,pt %ncc, 2b
add %o0, 0x100, %o0
3:
! ... check if 64 bytes to set
cmp %o4, 0x40
blu %ncc, .bzero_blk_done
nop
4:
stxa %g0, [%o0+0x0]%asi
stxa %g0, [%o0+0x8]%asi
stxa %g0, [%o0+0x10]%asi
stxa %g0, [%o0+0x18]%asi
stxa %g0, [%o0+0x20]%asi
stxa %g0, [%o0+0x28]%asi
stxa %g0, [%o0+0x30]%asi
stxa %g0, [%o0+0x38]%asi
subcc %o4, 0x40, %o4
bgu,pt %ncc, 3b
add %o0, 0x40, %o0
.bzero_blk_done:
membar #Sync
.bzero_small:
! Set the remaining doubles
subcc %o3, 8, %o3 ! Can we store any doubles?
blu,pn %ncc, .byteclr
and %o1, 7, %o1 ! calc bytes left after doubles
.dbclr:
stxa %g0, [%o0]%asi ! Clear the doubles
subcc %o3, 8, %o3
bgeu,pt %ncc, .dbclr
add %o0, 8, %o0
ba .byteclr
nop
.wdalign:
andcc %o0, 3, %o3 ! is add aligned on a word boundary
bz,pn %ncc, .wdclr
andn %o1, 3, %o3 ! create word sized count in %o3
dec %o1 ! decrement count
stba %g0, [%o0]%asi ! clear a byte
ba .wdalign
inc %o0 ! next byte
.wdclr:
sta %g0, [%o0]%asi ! 4-byte clearing loop
subcc %o3, 4, %o3
bnz,pt %ncc, .wdclr
inc 4, %o0
and %o1, 3, %o1 ! leftover count, if any
.byteclr:
! Set the leftover bytes
brz %o1, .bzero_exit
nop
7:
deccc %o1 ! byte clearing loop
stba %g0, [%o0]%asi
bgu,pt %ncc, 7b
inc %o0
.bzero_exit:
!
!
1:
#endif /* lint */
#ifdef ROCK_CR_6654578
/* This code tries to maximize bandwidth by being clever about accessing
* the two cache lines that are BUDDY PAIRS in the L3 cache. When line 0
* of a pair is accessed, it will take hundreds of cycles to get the line
* from memory, which brings in a 128-byte line to L3. Until the line is
* installed in L3, any other access to that line (such as buddy line 1)
* is blocked. For best throughput, we access many lines that are the first
* of their buddy pairs, and only after many such accesses have been made,
* we access the sequence of second buddy pair lines. Hopefully the second
* set of accesses comes after the L3 lines are installed, so the accesses
* hitin L3 without being delayed. This should yield better throughput.
* To keep this code simple, we assume the addresses given are aligned at
* least on a 128 byte boundary, and the length is assumed to be a multiple
* of 8k bytes.
*/
#ifdef lint
/*ARGSUSED*/
int
{
return(0);
}
#else /* lint */
.align 64
2:
#endif /* lint */
#endif /* ROCK_CR_6654578 */
#if defined(lint)
int use_hw_bcopy = 1;
int use_hw_bzero = 1;
#else /* !lint */
.word 1
.word 1
.word 0x100
.word 0x200
.word 0x400
.word 0x400
.align 64
.section ".text"
#endif /* !lint */