/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
*/
#include <sys/asm_linkage.h>
#include <sys/machthread.h>
#include <sys/privregs.h>
#include <sys/niagaraasi.h>
#if !defined(lint)
#include "assym.h"
#endif /* lint */
/*
* Pseudo-code to aid in understanding the control flow of the
*
* ! WARNING : <Register usage convention>
* ! In kcopy() the %o5, holds previous error handler and a flag
* ! LOFAULT_SET (low bits). The %o5 is null in bcopy().
* ! The %o5 is not available for any other use.
*
* On entry:
* ! Determine whether to use the FP register version or the
* ! the leaf routine version depending on the size of the copy.
* ! Set up error handling accordingly.
* ! The transition point depends on FP_COPY
* ! For both versions %o5 is reserved
*
* kcopy():
* if(length > FP_COPY)
* go to regular_kcopy
*
* ! Setup_leaf_rtn_error_handler
* %o5 = curthread->t_lofault; ! save existing handler in %o5
* %o5 |= LOFAULT_SET; ! ORed with LOFAULT_SET flag
* curthread->t_lofault = .sm_copyerr;
* goto small_bcopy();
*
* regular_kcopy:
* save_registers()
* %o5 = curthread->t_lofault; ! save existing handler in %o5
* %o5 |= LOFAULT_SET; ! ORed with LOFAULT_SET flag
* curthread->t_lofault = .copyerr;
* goto do_copy();
*
* bcopy():
* if(length > FP_COPY)
* go to regular_bcopy
*
* ! Setup_leaf_rtn_error_handler
* %o5 = curthread->t_lofault; ! save existing handler in %o5
* curthread->t_lofault = .sm_copyerr;
* goto small_bcopy();
*
* regular_bcopy:
* %o5 = curthread->t_lofault; ! save existing handler in %o5
* curthread->t_lofault = .copyerr;
* goto do_copy();
*
* small_bcopy:
* ! handle copies smaller than FP_COPY
* restore t_lofault handler
* exit
*
* do_copy:
* ! handle copies larger than FP_COPY
* save fp_regs
* blockcopy;
* restore fp_regs
* restore t_lofault handler if came from kcopy();
*
*
* In leaf lofault handler:
* curthread->t_lofault = (%o5 & ~LOFAULT_SET); ! restore old t_lofault
* return (errno)
*
* In lofault handler:
* curthread->t_lofault = (%o5 & ~LOFAULT_SET); ! restore old t_lofault
* restore fp_regs
* return (errno)
*
*
*
* to how the src and dst is aligned and how much data needs to be moved.
*
*
* if (count < FP_COPY) { (584 bytes)
* if count < SHORTCOPY (7 bytes)
* copy bytes; go to short_exit
* else
* get dst aligned on long word boundary
* if( src is on long word boundary ) {
* copy final 0-31 bytes; go to short_exit
* if src is half word aligned, ld half, ld word, ld half; pack
* into long word, store long words in 32-byte chunks
* if src is byte aligned, ld byte,half,word parts; pack into long
* word, store long words in 32-byte chunks
* move final 0-31 bytes according to src alignment; go to short_exit
* short_exit:
* restore trap handler if needed, retl
* else { More than FP_COPY bytes
* set fault handler
* disable kernel preemption
* save registers, save FP registers if in use
* move bytes to align destination register on long word boundary
* align dst on 64 byte boundary; use 8-way test for each of 8 possible
* src alignments relative to a 64 byte boundary to select the
* 16-way unrolled loop (128 bytes) to use for
* block load, fmovd, block-init-store, block-store, fmovd operations
* then go to remain_stuff.
* remain_stuff: move remaining bytes. go to long_exit
* } else {
* setup alignaddr for faligndata instructions
* align dst on 64 byte boundary; use 8-way test for each of 8 possible
* src alignments to nearest long word relative to 64 byte boundary to
* select the 8-way unrolled loop (64 bytes) to use for
* block load, falign, fmovd, block-store loop
* goto unalign_done.
* unalign_done:
* move remaining bytes for unaligned cases. go to long_exit
* long_exit:
* restore %gsr, FP regs (either from stack or set to zero),
* restore trap handler, check for kernel preemption request,
* handle if needed, ret.
* }
*
* Other platforms include hw_bcopy_limit_[1248] to control the exact
* point where the FP register code is used. On those platforms, the
* FP register code did not leave data in L2 cache, potentially affecting
* of the FP registers has no effect on L2 cache behavior.
* The cost for testing hw_bcopy_limit_* according to different
* alignments exceeds 50 cycles for all cases, even when hw_bcopy_limits
* were not used. That cost was judged too high relative to the benefits,
* so the hw_bcopy_limit option is omitted from this code.
*/
/*
* Less then or equal this number of bytes we will always copy byte-for-byte
*/
/*
* LOFAULT_SET : Flag set by kzero and kcopy to indicate that t_lofault
* handler was set
*/
/*
* This define is to align data for the unaligned source cases.
* The data1, data2 and data3 is merged into data1 and data2.
* The data3 is preserved for next merge.
*/
/*
* This macro is to align the data. Basically it merges
* data1 and data2 to form double word.
*/
#if !defined(NIAGARA_IMPL)
/*
* Flags set in the lower bits of the t_lofault address:
* FPUSED_FLAG: The FP registers were in use and must be restored
* LOFAULT_SET: Set for bcopy calls, cleared for kcopy calls
* COPY_FLAGS: Both of the above
*
* Other flags:
* KPREEMPT_FLAG: kpreempt needs to be called
*/
#define ALIGN_OFF_1_7 \
#define ALIGN_OFF_8_15 \
#define ALIGN_OFF_16_23 \
#define ALIGN_OFF_24_31 \
#define ALIGN_OFF_32_39 \
#define ALIGN_OFF_40_47 \
#define ALIGN_OFF_48_55 \
#define ALIGN_OFF_56_63 \
/*
* FP_COPY indicates the minimum number of bytes needed
* to justify using FP/VIS-accelerated memory operations.
* The FPBLK code assumes a minimum number of bytes are available
* to be moved on entry. Check that code carefully before
* reducing FP_COPY below 256.
*/
/*
* Size of stack frame in order to accomodate a 64-byte aligned
* floating-point register save area and 2 64-bit temp locations.
* All copy functions use three quadrants of fp registers; to assure a
* block-aligned three block buffer in which to save we must reserve
* four blocks on stack.
*
* _______________________________________ <-- %fp + STACK_BIAS
* | We may need to preserve 3 quadrants |
* | of fp regs, but since we do so with |
* | align to VIS_BLOCKSIZE bytes. So |
* | this area is 4 * VIS_BLOCKSIZE. | <-- - SAVED_FPREGS_OFFSET
* |-------------------------------------|
* | 8 bytes to save %fprs | <-- - SAVED_FPRS_OFFSET
* |-------------------------------------|
* | 8 bytes to save %gsr | <-- - SAVED_GSR_OFFSET
* ---------------------------------------
*/
/*
* In FP copies if we do not have preserved data to restore over
* the fp regs we used then we must zero those regs to avoid
* exposing portions of the data to later threads (data security).
*/
#define FZERO \
#if !defined(lint)
/*
* Used to save and restore in-use fp registers when we want to use FP.
*/
/* membar #Sync */ ;\
/* membar #Sync - provided at copy completion */ ;\
#endif /* NIAGARA_IMPL */
#endif /* lint */
/*
* Copy a block of storage, returning an error code if `from' or
* `to' takes a kernel pagefault which cannot be resolved.
* Returns errno value on pagefault error, 0 if all ok
*/
#if defined(lint)
/* ARGSUSED */
int
{ return(0); }
#else /* lint */
.seg ".text"
.align 4
#if !defined(NIAGARA_IMPL)
! t_lofault.
! t_lofault.
/*
* We got here because of a fault during a small kcopy or bcopy.
* if a fault handler existed when bcopy was called.
* No floating point registers are used by the small copies.
* Small copies are from a leaf routine
* Errno value is in %g1.
*/
! we're expected to just return the error code and not to
! invoke any existing error handler. As far as bcopy is concerned,
! we only set t_lofault if there was an existing lofault handler.
3:
! We're here via bcopy. There must have been an error handler
! in place otherwise we would have died a nasty death already.
jmp %o5 ! goto real handler
mov %g0, %o0
/*
* end of .sm_copyerr
*/
/*
* We got here because of a fault during kcopy or bcopy if a fault
* handler existed when bcopy was called.
* stack and fp registers need to be restored
* Errno value is in %g1.
*/
.copyerr:
sethi %hi(.copyerr2), %l1
or %l1, %lo(.copyerr2), %l1
membar #Sync ! sync error barrier
stn %l1, [THREAD_REG + T_LOFAULT] ! set t_lofault
btst FPUSED_FLAG, %o5
bz,pt %xcc, 1f
and %o5, LOFAULT_SET, %l1 ! copy flag to %l1
membar #Sync ! sync error barrier
wr %l5, 0, %gsr
btst FPRS_FEF, %g5
bz,pt %icc, 4f
nop
! restore fpregs from stack
BLD_FP_FROMSTACK(%o2)
ba,pt %ncc, 2f
wr %g5, 0, %fprs ! restore fprs
4:
FZERO
wr %g5, 0, %fprs ! restore fprs
2:
ldn [THREAD_REG + T_LWP], %o2
brnz,pt %o2, 1f
nop
ldsb [THREAD_REG + T_PREEMPT], %l0
deccc %l0
bnz,pn %ncc, 1f
stb %l0, [THREAD_REG + T_PREEMPT]
! Check for a kernel preemption request
ldn [THREAD_REG + T_CPU], %l0
ldub [%l0 + CPU_KPRUNRUN], %l0
brnz,a,pt %l0, 1f ! Need to call kpreempt?
or %l1, KPREEMPT_FLAG, %l1 ! If so, set the flag
! The kcopy will always set a t_lofault handler. If it fires,
! handler after resetting the t_lofault value.
1:
andn %o5, COPY_FLAGS, %o5 ! remove flags from lofault address
membar #Sync ! sync error barrier
stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
! call kpreempt if necessary
btst KPREEMPT_FLAG, %l1
bz,pt %icc, 2f
nop
call kpreempt
rdpr %pil, %o0 ! pass %pil
2:
btst LOFAULT_SET, %l1
bnz,pn %ncc, 3f
nop
ret
restore %g1, 0, %o0
3:
/*
* We got here because of a fault in .copyerr. We can't safely restore fp
* state, so we panic.
*/
.asciz "Unable to restore fp state after copy operation"
.align 4
.copyerr2:
/*
* end of .copyerr
*/
#else /* NIAGARA_IMPL */
/*
* We got here because of a fault during kcopy.
* Errno value is in %g1.
*/
.copyerr:
#endif /* NIAGARA_IMPL */
#endif /* lint */
/*
* Copy a block of storage - must not overlap (from + len <= to).
*/
#if defined(lint)
/* ARGSUSED */
void
{}
#else /* lint */
#if !defined(NIAGARA_IMPL)
/*
* Special case for handling when src and dest are both long word aligned
* and total data to move is less than FP_COPY bytes
* Also handles finish up for large block moves, so may be less than 32 bytes
*/
.bc_medl8:
.bc_medl7:
nop !
/*
* Align destination to long word boundary
*/
/*
* Handle all cases where src and dest are aligned on word
* boundaries. Use unrolled loops for better performance.
* This option wins over standard large data move when
* source and destination is in cache for medium
* to short data moves.
*/
nop !
.bc_medw7:
nop !
nop !
.bc_medh7:
nop !
.align 16
nop !
.bc_medb7:
nop !
.align 16
nop !
/*
* The _more entry points are not intended to be used directly by
* any caller from outside this file. They are provided to allow
* profiling and dtrace of the portions of the copy code that uses
* the floating point registers.
*/
! We've already captured whether t_lofault was zero on entry.
! We need to mark ourselves as being from bcopy since both
! kcopy and bcopy use the same code path. If LOFAULT_SET is
! returning.
.do_copy:
/*
* kpreempt_disable();
*/
1:
/*
* Following code is for large copies. We know there is at
* least FP_COPY bytes available. FP regs are used, so
* we save registers and fp regs before starting
*/
.bc_aln_1:
nop !
.bc_aln_7:
nop !
sub %i3, 64, %i3 ! end of source buffer
andn %i0, 0x3f, %o4 ! %o4 has block aligned src address
prefetch [%o4 + (3 * CACHE_LINE)], #one_read
alignaddr %i0, %g0, %g0 ! generate %gsr
add %i0, %i3, %i0 ! advance %i0 to after blocks
!
! Determine source alignment to correct 8 byte offset
andcc %i0, 0x20, %o3
brnz,pn %o3, .bc_unaln_1
andcc %i0, 0x10, %o3
brnz,pn %o3, .bc_unaln_01
andcc %i0, 0x08, %o3
brz,a %o3, .bc_unaln_000
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_001
nop
.bc_unaln_01:
brnz,a %o3, .bc_unaln_011
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_010
nop
.bc_unaln_1:
brnz,pn %o3, .bc_unaln_11
andcc %i0, 0x08, %o3
brnz,a %o3, .bc_unaln_101
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_100
nop
.bc_unaln_11:
brz,pn %o3, .bc_unaln_110
prefetch [%i0 + (4 * CACHE_LINE)], #one_read
.bc_unaln_111:
ldd [%o4+56], %d14
.bc_unaln_111_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d14, %d16, %d48
faligndata %d16, %d18, %d50
faligndata %d18, %d20, %d52
faligndata %d20, %d22, %d54
faligndata %d22, %d24, %d56
faligndata %d24, %d26, %d58
faligndata %d26, %d28, %d60
faligndata %d28, %d30, %d62
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_111_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_done
nop
.bc_unaln_110:
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.bc_unaln_110_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d12, %d14, %d48
faligndata %d14, %d16, %d50
faligndata %d16, %d18, %d52
faligndata %d18, %d20, %d54
faligndata %d20, %d22, %d56
faligndata %d22, %d24, %d58
faligndata %d24, %d26, %d60
faligndata %d26, %d28, %d62
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_110_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_done
nop
.bc_unaln_101:
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.bc_unaln_101_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d10, %d12, %d48
faligndata %d12, %d14, %d50
faligndata %d14, %d16, %d52
faligndata %d16, %d18, %d54
faligndata %d18, %d20, %d56
faligndata %d20, %d22, %d58
faligndata %d22, %d24, %d60
faligndata %d24, %d26, %d62
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_101_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_done
nop
.bc_unaln_100:
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.bc_unaln_100_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d8, %d10, %d48
faligndata %d10, %d12, %d50
faligndata %d12, %d14, %d52
faligndata %d14, %d16, %d54
faligndata %d16, %d18, %d56
faligndata %d18, %d20, %d58
faligndata %d20, %d22, %d60
faligndata %d22, %d24, %d62
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_100_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_done
nop
.bc_unaln_011:
ldd [%o4+24], %d6
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.bc_unaln_011_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d6, %d8, %d48
faligndata %d8, %d10, %d50
faligndata %d10, %d12, %d52
faligndata %d12, %d14, %d54
faligndata %d14, %d16, %d56
faligndata %d16, %d18, %d58
faligndata %d18, %d20, %d60
faligndata %d20, %d22, %d62
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_011_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_done
nop
.bc_unaln_010:
ldd [%o4+16], %d4
ldd [%o4+24], %d6
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.bc_unaln_010_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d4, %d6, %d48
faligndata %d6, %d8, %d50
faligndata %d8, %d10, %d52
faligndata %d10, %d12, %d54
faligndata %d12, %d14, %d56
faligndata %d14, %d16, %d58
faligndata %d16, %d18, %d60
faligndata %d18, %d20, %d62
fmovd %d20, %d4
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_010_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_done
nop
.bc_unaln_001:
ldd [%o4+8], %d2
ldd [%o4+16], %d4
ldd [%o4+24], %d6
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.bc_unaln_001_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d2, %d4, %d48
faligndata %d4, %d6, %d50
faligndata %d6, %d8, %d52
faligndata %d8, %d10, %d54
faligndata %d10, %d12, %d56
faligndata %d12, %d14, %d58
faligndata %d14, %d16, %d60
faligndata %d16, %d18, %d62
fmovd %d18, %d2
fmovd %d20, %d4
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_001_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .bc_unaln_done
nop
.bc_unaln_000:
ldda [%o4]ASI_BLK_P, %d0
.bc_unaln_000_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d0, %d2, %d48
faligndata %d2, %d4, %d50
faligndata %d4, %d6, %d52
faligndata %d6, %d8, %d54
faligndata %d8, %d10, %d56
faligndata %d10, %d12, %d58
faligndata %d12, %d14, %d60
faligndata %d14, %d16, %d62
fmovd %d16, %d0
fmovd %d18, %d2
fmovd %d20, %d4
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_P
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .bc_unaln_000_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
.bc_unaln_done:
! Handle trailing bytes, 64 to 127
! Dest long word aligned, Src not long word aligned
cmp %i2, 15
bleu %ncc, .bc_unaln_short
andn %i2, 0x7, %i3 ! %i3 is multiple of 8
and %i2, 0x7, %i2 ! residue bytes in %i2
add %i2, 8, %i2
.bc_exit:
.bc_ex2:
1:
2:
3:
#else /* NIAGARA_IMPL */
.do_copy:
1:
/*
* Compare against 256 since we should be checking block addresses
* and (dest & ~63) - (src & ~63) can be 3 blocks even if
* src = dest + (64 * 3) + 63.
*/
/*
* Copy that reach here have at least 2 blocks of data to copy.
*/
! 1B aligned
.alwdcp:
.alewdcp:
.chksrc:
! partial
.blkcpy:
1:
.blkdone:
.last8:
.last4:
1:
.last2:
1:
.residue:
.blkexit:
1:
.bcb_punt:
!
!
!
!
!
!
bnz,a 1b
1:
bnz,a 1b
b 2f
b 3f
!
!
1:
bnz,a 1b
.xfer:
3:
2:
b 2b ! loop
1:
!
!
b,a .xfer
!
!
.aldoubcp:
!
!
5:
!
!
.wordcp:
5:
b,a .dbytecp
.alwordcp:
b .wordcp
!
!
.bytecp:
b .dbytecp
!
!
1:
.dbytecp:
.cpdone:
1:
/*
* Common code used to align transfers on word and doubleword
* boundaries. Aligns source and destination and returns a count
* of aligned bytes to transfer in %i3
*/
1:
.alignit:
bnz,a 1b
#endif /* NIAGARA_IMPL */
#endif /* lint */
/*
* Block copy with possibly overlapped operands.
*/
#if defined(lint)
/*ARGSUSED*/
void
{}
#else /* lint */
retl ! return
1:
!
!
.ov_fwd:
retl ! return
!
!
.ov_bkwd:
retl ! return
#endif /* lint */
/*
* hwblkpagecopy()
*
* Copies exactly one page. This routine assumes the caller (ppcopy)
* has already disabled kernel preemption and has checked
* use_hw_bcopy.
*/
#ifdef lint
/*ARGSUSED*/
void
{ }
#else /* lint */
/*
* Copying exactly one page and PAGESIZE is in mutliple of 0x80.
*/
1:
#endif /* lint */
/*
* Transfer data to and from user space -
* Note that these routines can cause faults
* It is assumed that the kernel has nothing at
* less than KERNELBASE in the virtual address space.
*
* Note that copyin(9F) and copyout(9F) are part of the
*
* Sigh.
*
* So there's two extremely similar routines - xcopyin() and xcopyout()
* which return the errno that we've faithfully computed. This
* allows other callers (e.g. uiomove(9F)) to work correctly.
* Given that these are used pretty heavily, we expand the calling
* sequences inline for all flavours (rather than making wrappers).
*
* There are also stub routines for xcopyout_little and xcopyin_little,
* which currently are intended to handle requests of <= 16 bytes from
* do_unaligned. Future enhancement to make them handle 8k pages efficiently
* is left as an exercise...
*/
/*
* Copy user data to kernel space (copyOP/xcopyOP/copyOP_noerr)
*
* General theory of operation:
*
* None of the copyops routines grab a window until it's decided that
* we need to do a HW block copy operation. This saves a window
*
* This code uses a set of 4 limits for the maximum size that will
* the default limits are:
*
* single byte aligned - 256 (hw_copy_limit_1)
* two byte aligned - 512 (hw_copy_limit_2)
* four byte aligned - 1024 (hw_copy_limit_4)
* eight byte aligned - 1024 (hw_copy_limit_8)
*
* If the value for a particular limit is zero, the copy will be done
*
* Flow:
*
* If count == zero return zero.
*
* Store the previous lo_fault handler into %g6.
* Place our secondary lofault handler into %g5.
* Place the address of our nowindow fault handler into %o3.
* Place the address of the windowed fault handler into %o4.
* --> We'll use this handler if we end up grabbing a window
* --> before we use block initializing store and quad load ASIs
*
* If count is less than or equal to SMALL_LIMIT (7) we
* always do a byte for byte copy.
*
* If count is > SMALL_LIMIT, we check the alignment of the input
* and output pointers. Based on the alignment we check count
* against a limit based on detected alignment. If we exceed the
* alignment value we copy via block initializing store and quad
* load instructions.
*
* If we don't exceed one of the limits, we store -count in %o3,
* we store the number of chunks (8, 4, 2 or 1 byte) operated
* on in our basic copy loop in %o2. Following this we branch
* to the appropriate copy loop and copy that many chunks.
* Since we've been adding the chunk size to %o3 each time through
* as well as decrementing %o2, we can tell if any data is
* is left to be copied by examining %o3. If that is zero, we're
* done and can go home. If not, we figure out what the largest
* chunk size left to be copied is and branch to that copy loop
* unless there's only one byte left. We load that as we're
* branching to code that stores it just before we return.
*
* Fault handlers are invoked if we reference memory that has no
* current mapping. All forms share the same copyio_fault handler.
* This routine handles fixing up the stack and general housecleaning.
* Each copy operation has a simple fault handler that is then called
* to do the work specific to the invidual operation. The handler
* for copyOP and xcopyOP are found at the end of individual function.
* The handlers for xcopyOP_little are found at the end of xcopyin_little.
* The handlers for copyOP_noerr are found at the end of copyin_noerr.
*/
/*
* Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
*/
#if defined(lint)
/*ARGSUSED*/
int
{ return (0); }
#else /* lint */
/*
* We save the arguments in the following registers in case of a fault:
* kaddr - %g2
* uaddr - %g3
* count - %g4
*/
/*
* Generic copyio fault handler. This is the first line of defense when a
* fault occurs in (x)copyin/(x)copyout. In order for this to function
* properly, the value of the 'real' lofault handler should be in REAL_LOFAULT.
* This allows us to share common code for all the flavors of the copy
* operations, including the _noerr versions.
*
* Note that this function will restore the original input parameters before
* calling REAL_LOFAULT. So the real handler can vector to the appropriate
* member of the t_copyop structure, if needed.
*/
#if !defined(NIAGARA_IMPL)
bz 1f
4:
1:
#else /* NIAGARA_IMPL */
#endif /* NIAGARA_IMPL */
#if !defined(NIAGARA_IMPL)
/*
* Small copy out code
*
*/
/*
* Special case for handling when src and dest are both long word aligned
* and total data to move is less than FP_COPY bytes
* Also handles finish up for large block moves, so may be less than 32 bytes
*/
.co_medl8:
.co_medl7:
nop !
/*
* Handle all cases where src and dest are aligned on word
* boundaries. Use unrolled loops for better performance.
* This option wins over standard large data move when
* source and destination is in cache for medium
* to short data moves.
*/
nop !
.co_medw7:
nop !
nop !
.co_medh7:
nop !
.align 16
nop !
.co_medb7:
nop !
.align 16
nop !
/*
* End of small copy (no window) code
*/
/*
* Long copy code
*/
/*
* Following code is for large copies. We know there is at
* least FP_COPY bytes available. FP regs are used, so
* we save registers and fp regs before starting
*/
.co_aln_1:
nop !
.co_aln_7:
nop !
sub %i3, 64, %i3 ! end of source buffer
andn %i0, 0x3f, %o4 ! %o4 has block aligned src address
prefetch [%o4 + (3 * CACHE_LINE)], #one_read
alignaddr %i0, %g0, %g0 ! generate %gsr
add %i0, %i3, %i0 ! advance %i0 to after blocks
!
! Determine source alignment to correct 8 byte offset
andcc %i0, 0x20, %o3
brnz,pn %o3, .co_unaln_1
andcc %i0, 0x10, %o3
brnz,pn %o3, .co_unaln_01
andcc %i0, 0x08, %o3
brz,a %o3, .co_unaln_000
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_001
nop
.co_unaln_01:
brnz,a %o3, .co_unaln_011
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_010
nop
.co_unaln_1:
brnz,pn %o3, .co_unaln_11
andcc %i0, 0x08, %o3
brnz,a %o3, .co_unaln_101
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_100
nop
.co_unaln_11:
brz,pn %o3, .co_unaln_110
prefetch [%i0 + (4 * CACHE_LINE)], #one_read
.co_unaln_111:
ldd [%o4+56], %d14
.co_unaln_111_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d14, %d16, %d48
faligndata %d16, %d18, %d50
faligndata %d18, %d20, %d52
faligndata %d20, %d22, %d54
faligndata %d22, %d24, %d56
faligndata %d24, %d26, %d58
faligndata %d26, %d28, %d60
faligndata %d28, %d30, %d62
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_111_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_done
nop
.co_unaln_110:
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.co_unaln_110_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d12, %d14, %d48
faligndata %d14, %d16, %d50
faligndata %d16, %d18, %d52
faligndata %d18, %d20, %d54
faligndata %d20, %d22, %d56
faligndata %d22, %d24, %d58
faligndata %d24, %d26, %d60
faligndata %d26, %d28, %d62
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_110_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_done
nop
.co_unaln_101:
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.co_unaln_101_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d10, %d12, %d48
faligndata %d12, %d14, %d50
faligndata %d14, %d16, %d52
faligndata %d16, %d18, %d54
faligndata %d18, %d20, %d56
faligndata %d20, %d22, %d58
faligndata %d22, %d24, %d60
faligndata %d24, %d26, %d62
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_101_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_done
nop
.co_unaln_100:
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.co_unaln_100_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d8, %d10, %d48
faligndata %d10, %d12, %d50
faligndata %d12, %d14, %d52
faligndata %d14, %d16, %d54
faligndata %d16, %d18, %d56
faligndata %d18, %d20, %d58
faligndata %d20, %d22, %d60
faligndata %d22, %d24, %d62
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_100_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_done
nop
.co_unaln_011:
ldd [%o4+24], %d6
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.co_unaln_011_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d6, %d8, %d48
faligndata %d8, %d10, %d50
faligndata %d10, %d12, %d52
faligndata %d12, %d14, %d54
faligndata %d14, %d16, %d56
faligndata %d16, %d18, %d58
faligndata %d18, %d20, %d60
faligndata %d20, %d22, %d62
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_011_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_done
nop
.co_unaln_010:
ldd [%o4+16], %d4
ldd [%o4+24], %d6
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.co_unaln_010_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d4, %d6, %d48
faligndata %d6, %d8, %d50
faligndata %d8, %d10, %d52
faligndata %d10, %d12, %d54
faligndata %d12, %d14, %d56
faligndata %d14, %d16, %d58
faligndata %d16, %d18, %d60
faligndata %d18, %d20, %d62
fmovd %d20, %d4
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_010_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_done
nop
.co_unaln_001:
ldd [%o4+8], %d2
ldd [%o4+16], %d4
ldd [%o4+24], %d6
ldd [%o4+32], %d8
ldd [%o4+40], %d10
ldd [%o4+48], %d12
ldd [%o4+56], %d14
.co_unaln_001_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d2, %d4, %d48
faligndata %d4, %d6, %d50
faligndata %d6, %d8, %d52
faligndata %d8, %d10, %d54
faligndata %d10, %d12, %d56
faligndata %d12, %d14, %d58
faligndata %d14, %d16, %d60
faligndata %d16, %d18, %d62
fmovd %d18, %d2
fmovd %d20, %d4
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_001_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
ba .co_unaln_done
nop
.co_unaln_000:
ldda [%o4]ASI_BLK_P, %d0
.co_unaln_000_loop:
add %o4, 64, %o4
ldda [%o4]ASI_BLK_P, %d16
faligndata %d0, %d2, %d48
faligndata %d2, %d4, %d50
faligndata %d4, %d6, %d52
faligndata %d6, %d8, %d54
faligndata %d8, %d10, %d56
faligndata %d10, %d12, %d58
faligndata %d12, %d14, %d60
faligndata %d14, %d16, %d62
fmovd %d16, %d0
fmovd %d18, %d2
fmovd %d20, %d4
fmovd %d22, %d6
fmovd %d24, %d8
fmovd %d26, %d10
fmovd %d28, %d12
fmovd %d30, %d14
stda %d48, [%i1]ASI_BLK_AIUS
subcc %i3, 64, %i3
add %i1, 64, %i1
bgu,pt %ncc, .co_unaln_000_loop
prefetch [%o4 + (4 * CACHE_LINE)], #one_read
.co_unaln_done:
! Handle trailing bytes, 64 to 127
! Dest long word aligned, Src not long word aligned
cmp %i2, 15
bleu %ncc, .co_unaln_short
andn %i2, 0x7, %i3 ! %i3 is multiple of 8
and %i2, 0x7, %i2 ! residue bytes in %i2
add %i2, 8, %i2
.co_exit:
.co_ex2:
2:
#else /* NIAGARA_IMPL */
!
!
1:
!
! Run in leaf mode, using the %o regs as our input regs.
!
subcc %o2, SMALL_LIMIT, %o3
bgu,a,pt %ncc, .dco_ns
or %o0, %o1, %o3
!
! What was previously ".small_copyout"
! Do full differenced copy.
!
.dcobcp:
sub %g0, %o2, %o3 ! negate count
add %o0, %o2, %o0 ! make %o0 point at the end
add %o1, %o2, %o1 ! make %o1 point at the end
ba,pt %ncc, .dcocl
ldub [%o0 + %o3], %o4 ! load first byte
!
! %o0 and %o2 point at the end and remain pointing at the end
! of their buffers. We pull things out by adding %o3 (which is
! the negation of the length) to the buffer end which gives us
! the curent location in the buffers. By incrementing %o3 we walk
! through both buffers without having to bump each buffer's
!
.align 16
.dcocl:
!
! We're done. Go home.
!
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT]
retl
clr %o0
!
! Try aligned copies from here.
!
.dco_ns:
! %o0 = kernel addr (to be copied from)
! %o1 = user addr (to be copied to)
! %o2 = length
! %o3 = %o1 | %o2 (used for alignment checking)
! %o4 is alternate lo_fault
! %o5 is original lo_fault
!
! bounce to the byte for byte copy loop. Otherwise do it in
! HW (if enabled).
!
btst 1, %o3
bz,pt %icc, .dcoh8
btst 7, %o3
!
! Single byte aligned. Do we do it via HW or via
! byte for byte? Do a quick no memory reference
! check to pick up small copies.
!
sethi %hi(hw_copy_limit_1), %o3
!
! Big enough that we need to check the HW limit for
! this size copy.
!
ld [%o3 + %lo(hw_copy_limit_1)], %o3
!
! Is HW copy on? If not, do everything byte for byte.
!
tst %o3
bz,pn %icc, .dcobcp
subcc %o3, %o2, %o3
!
!
!
! We're big enough and copy is on. Do it with HW.
!
ba,pt %ncc, .big_copyout
nop
.dcoh8:
!
! 8 byte aligned?
!
bnz,a %ncc, .dcoh4
btst 3, %o3
!
!
!
!
!
! We're negative if our size is larger than hw_copy_limit_8.
!
bge,pt %ncc, .dcos8
nop
!
!
.dcos8:
!
!
!
!
.dcoh4:
!
! If so, go off an do the copy.
! If not, load the hard limit. %o3 is
! available for reuse.
!
sethi %hi(hw_copy_limit_4), %o3
ld [%o3 + %lo(hw_copy_limit_4)], %o3
!
! Bop off to the aligned copy.
!
tst %o3
bz,pn %icc, .dcos4
subcc %o3, %o2, %o3
!
!
!
!
ba,pt %ncc, .big_copyout
nop
.dcos4:
add %o0, %o2, %o0
add %o1, %o2, %o1
sub %g0, %o2, %o3
ba,pt %ncc, .dodfbc
srl %o2, 2, %o2 ! Number of 4 byte chunks to copy
!
! We must be 2 byte aligned. Off we go.
! The check for small copies was done in the
! delay at .dcoh4
!
.dcoh2:
ble %ncc, .dcos2
sethi %hi(hw_copy_limit_2), %o3
ld [%o3 + %lo(hw_copy_limit_2)], %o3
tst %o3
bz,pn %icc, .dcos2
subcc %o3, %o2, %o3
bge,pt %ncc, .dcos2
nop
!
!
.dcos2:
!
!
!
.align 32
.dodebc:
!
! eight byte aligned copies end here.
!
bz,pt %ncc, .dcofh
nop
!
! Something is left - do it byte for byte.
!
ba,pt %ncc, .dcocl
ldub [%o0 + %o3], %o4 ! load next byte
!
! Four byte copy loop. %o2 is the number of 4 byte chunks to copy.
!
.align 32
.dodfbc:
lduw [%o0 + %o3], %o4
deccc %o2
sta %o4, [%o1 + %o3]ASI_USER
bg,pt %ncc, .dodfbc
addcc %o3, 4, %o3
!
!
!
!
!
! copy.
!
.align 32
.dodtbc:
!
!
!
!
.dcofh:
! We're going to go off and do a block copy.
! Switch fault handlers and grab a window. We
! kernel data to this point.
stn %o4, [THREAD_REG + T_LOFAULT]
! Copy out that reach here are larger than 256 bytes. The
! hw_copy_limit_1 is set to 256. Never set this limit less
! 128 bytes.
save %sp, -SA(MINFRAME), %sp
.do_block_copyout:
mov %i1, %i5
mov %i0, %i1
mov %i5, %i0
! Block (64 bytes) align the destination.
andcc %i0, 0x3f, %i3 ! is dst block aligned
bz %ncc, copyout_blalign ! dst already block aligned
sub %i3, 0x40, %i3
neg %i3 ! bytes till dst 64 bytes aligned
sub %i2, %i3, %i2 ! update i2 with new count
! Based on source and destination alignment do
! either 8 bytes, 4 bytes, 2 bytes or byte copy.
! Is dst & src 8B aligned
or %i0, %i1, %o2
andcc %o2, 0x7, %g0
bz %ncc, .co_alewdcp
nop
! Is dst & src 4B aligned
andcc %o2, 0x3, %g0
bz %ncc, .co_alwdcp
nop
! Is dst & src 2B aligned
andcc %o2, 0x1, %g0
bz %ncc, .co_alhlfwdcp
nop
! 1B aligned
1: ldub [%i1], %o2
stba %o2, [%i0]ASI_USER
inc %i1
deccc %i3
bgu,pt %ncc, 1b
inc %i0
ba copyout_blalign
nop
! dst & src 4B aligned
.co_alwdcp:
ld [%i1], %o2
sta %o2, [%i0]ASI_USER
add %i1, 0x4, %i1
subcc %i3, 0x4, %i3
bgu,pt %ncc, .co_alwdcp
add %i0, 0x4, %i0
ba copyout_blalign
nop
! dst & src 2B aligned
.co_alhlfwdcp:
lduh [%i1], %o2
stuha %o2, [%i0]ASI_USER
add %i1, 0x2, %i1
subcc %i3, 0x2, %i3
bgu,pt %ncc, .co_alhlfwdcp
add %i0, 0x2, %i0
ba copyout_blalign
nop
! dst & src 8B aligned
.co_alewdcp:
ldx [%i1], %o2
stxa %o2, [%i0]ASI_USER
add %i1, 0x8, %i1
subcc %i3, 0x8, %i3
bgu,pt %ncc, .co_alewdcp
add %i0, 0x8, %i0
! Now Destination is block (64 bytes) aligned
copyout_blalign:
andn %i2, 0x3f, %i3 ! %i3 count is multiple of block size
sub %i2, %i3, %i2 ! Residue bytes in %i2
mov ASI_BLK_INIT_QUAD_LDD_AIUS, %asi
andcc %i1, 0xf, %o2 ! is src quadword aligned
bz,pn %xcc, .co_blkcpy ! src offset in %o2 (last 4-bits)
nop
cmp %o2, 0x8
bg .co_upper_double
nop
bl .co_lower_double
nop
! Falls through when source offset is equal to 8 i.e.
! source is double word aligned.
sub %i1, %o2, %i1 ! align the src at 16 bytes.
andn %i1, 0x3f, %l0 ! %l0 has block aligned source
prefetch [%l0+0x0], #one_read
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
.co_loop0:
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l4
prefetch [%l0+0x40], #one_read
stxa %l3, [%i0+0x0]%asi
stxa %l4, [%i0+0x8]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
stxa %l5, [%i0+0x10]%asi
stxa %l2, [%i0+0x18]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l4
stxa %l3, [%i0+0x20]%asi
stxa %l4, [%i0+0x28]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
stxa %l5, [%i0+0x30]%asi
stxa %l2, [%i0+0x38]%asi
add %l0, 0x40, %l0
subcc %i3, 0x40, %i3
bgu,pt %xcc, .co_loop0
add %i0, 0x40, %i0
ba .co_blkdone
add %i1, %o2, %i1 ! increment the source by src offset
! the src offset was stored in %o2
.co_lower_double:
sub %i1, %o2, %i1 ! align the src at 16 bytes.
sll %o2, 3, %o0 ! %o0 left shift
mov 0x40, %o1
sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
andn %i1, 0x3f, %l0 ! %l0 has block aligned source
prefetch [%l0+0x0], #one_read
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2 ! partial data in %l2 and %l3 has
! complete data
.co_loop1:
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l4 ! %l4 has partial data
! for this read.
ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6) ! merge %l2, %l3 and %l4
! into %l2 and %l3
prefetch [%l0+0x40], #one_read
stxa %l2, [%i0+0x0]%asi
stxa %l3, [%i0+0x8]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6) ! merge %l2 with %l5 and
! %l4 from previous read
! into %l4 and %l5
stxa %l4, [%i0+0x10]%asi
stxa %l5, [%i0+0x18]%asi
! Repeat the same for next 32 bytes.
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l4
ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6)
stxa %l2, [%i0+0x20]%asi
stxa %l3, [%i0+0x28]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6)
stxa %l4, [%i0+0x30]%asi
stxa %l5, [%i0+0x38]%asi
add %l0, 0x40, %l0
subcc %i3, 0x40, %i3
bgu,pt %xcc, .co_loop1
add %i0, 0x40, %i0
ba .co_blkdone
add %i1, %o2, %i1 ! increment the source by src offset
! the src offset was stored in %o2
.co_upper_double:
sub %i1, %o2, %i1 ! align the src at 16 bytes.
sub %o2, 0x8, %o0
sll %o0, 3, %o0 ! %o0 left shift
mov 0x40, %o1
sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
andn %i1, 0x3f, %l0 ! %l0 has block aligned source
prefetch [%l0+0x0], #one_read
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2 ! partial data in %l3
! for this read and
! no data in %l2
.co_loop2:
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l4 ! %l4 has complete data
! and %l5 has partial
ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6) ! merge %l3, %l4 and %l5
! into %l3 and %l4
prefetch [%l0+0x40], #one_read
stxa %l3, [%i0+0x0]%asi
stxa %l4, [%i0+0x8]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6) ! merge %l2 and %l3 with
! %l5 from previous read
! into %l5 and %l2
stxa %l5, [%i0+0x10]%asi
stxa %l2, [%i0+0x18]%asi
! Repeat the same for next 32 bytes.
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l4
ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6)
stxa %l3, [%i0+0x20]%asi
stxa %l4, [%i0+0x28]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6)
stxa %l5, [%i0+0x30]%asi
stxa %l2, [%i0+0x38]%asi
add %l0, 0x40, %l0
subcc %i3, 0x40, %i3
bgu,pt %xcc, .co_loop2
add %i0, 0x40, %i0
ba .co_blkdone
add %i1, %o2, %i1 ! increment the source by src offset
! the src offset was stored in %o2
! Do fast copy using ASI_BLK_INIT_ST_QUAD_LDD_P
.co_blkcpy:
andn %i1, 0x3f, %o0 ! %o0 has block aligned source
prefetch [%o0+0x0], #one_read
1:
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l0
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l2
add %i1, 0x10, %i1
prefetch [%o0+0x40], #one_read
stxa %l0, [%i0+0x0]%asi
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l4
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_ST_QUAD_LDD_P, %l6
add %i1, 0x10, %i1
stxa %l1, [%i0+0x8]%asi
stxa %l2, [%i0+0x10]%asi
stxa %l3, [%i0+0x18]%asi
stxa %l4, [%i0+0x20]%asi
stxa %l5, [%i0+0x28]%asi
stxa %l6, [%i0+0x30]%asi
stxa %l7, [%i0+0x38]%asi
add %o0, 0x40, %o0
subcc %i3, 0x40, %i3
bgu,pt %xcc, 1b
add %i0, 0x40, %i0
.co_blkdone:
membar #Sync
brz,pt %i2, .copyout_exit
nop
! Handle trailing bytes
cmp %i2, 0x8
blu,pt %ncc, .co_residue
nop
! Can we do some 8B ops
or %i1, %i0, %o2
andcc %o2, 0x7, %g0
bnz %ncc, .co_last4
nop
! Do 8byte ops as long as possible
.co_last8:
ldx [%i1], %o2
stxa %o2, [%i0]ASI_USER
add %i1, 0x8, %i1
sub %i2, 0x8, %i2
cmp %i2, 0x8
bgu,pt %ncc, .co_last8
add %i0, 0x8, %i0
brz,pt %i2, .copyout_exit
nop
ba .co_residue
nop
.co_last4:
! Can we do 4B ops
andcc %o2, 0x3, %g0
bnz %ncc, .co_last2
nop
1:
ld [%i1], %o2
sta %o2, [%i0]ASI_USER
add %i1, 0x4, %i1
sub %i2, 0x4, %i2
cmp %i2, 0x4
bgu,pt %ncc, 1b
add %i0, 0x4, %i0
brz,pt %i2, .copyout_exit
nop
ba .co_residue
nop
.co_last2:
! Can we do 2B ops
andcc %o2, 0x1, %g0
bnz %ncc, .co_residue
nop
1:
lduh [%i1], %o2
stuha %o2, [%i0]ASI_USER
add %i1, 0x2, %i1
sub %i2, 0x2, %i2
cmp %i2, 0x2
bgu,pt %ncc, 1b
add %i0, 0x2, %i0
brz,pt %i2, .copyout_exit
nop
! Copy the residue as byte copy
.co_residue:
ldub [%i1], %i4
stba %i4, [%i0]ASI_USER
inc %i1
deccc %i2
bgu,pt %xcc, .co_residue
inc %i0
.copyout_exit:
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
ret
restore %g0, 0, %o0
.copyout_err:
ldn [THREAD_REG + T_COPYOPS], %o4
brz %o4, 2f
nop
ldn [%o4 + CP_COPYOUT], %g2
jmp %g2
nop
2:
retl
mov -1, %o0
#endif /* NIAGARA_IMPL */
SET_SIZE(copyout)
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
xcopyout(const void *kaddr, void *uaddr, size_t count)
{ return (0); }
#else /* lint */
ENTRY(xcopyout)
sethi %hi(.xcopyout_err), REAL_LOFAULT
b .do_copyout
or REAL_LOFAULT, %lo(.xcopyout_err), REAL_LOFAULT
.xcopyout_err:
ldn [THREAD_REG + T_COPYOPS], %o4
brz %o4, 2f
nop
ldn [%o4 + CP_XCOPYOUT], %g2
jmp %g2
nop
2:
retl
mov %g1, %o0
SET_SIZE(xcopyout)
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
xcopyout_little(const void *kaddr, void *uaddr, size_t count)
{ return (0); }
#else /* lint */
ENTRY(xcopyout_little)
sethi %hi(.little_err), %o4
ldn [THREAD_REG + T_LOFAULT], %o5
or %o4, %lo(.little_err), %o4
membar #Sync ! sync error barrier
stn %o4, [THREAD_REG + T_LOFAULT]
subcc %g0, %o2, %o3
add %o0, %o2, %o0
bz,pn %ncc, 2f ! check for zero bytes
sub %o2, 1, %o4
add %o1, %o2, %o1
ldub [%o0+%o3], %o4
1: stba %o4, [%o1+%o3]ASI_AIUSL
inccc %o3
sub %o0, 2, %o0 ! get next byte
bcc,a,pt %ncc, 1b
ldub [%o0+%o3], %o4
2: membar #Sync ! sync error barrier
stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
retl
mov %g0, %o0 ! return (0)
SET_SIZE(xcopyout_little)
#endif /* lint */
/*
* Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
*/
#if defined(lint)
/*ARGSUSED*/
int
copyin(const void *uaddr, void *kaddr, size_t count)
{ return (0); }
#else /* lint */
ENTRY(copyin)
sethi %hi(.copyin_err), REAL_LOFAULT
or REAL_LOFAULT, %lo(.copyin_err), REAL_LOFAULT
#if !defined(NIAGARA_IMPL)
.do_copyin:
tst %o2 ! check for zero count; quick exit
bz,pt %ncc, .ci_smallqx
mov %o0, SAVE_SRC
mov %o1, SAVE_DST
mov %o2, SAVE_COUNT
bgt,pt %ncc, .ci_copy_more
ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT
/*
* Small copy in code
*
*/
sethi %hi(copyio_fault_nowindow), %o3
or %o3, %lo(copyio_fault_nowindow), %o3
membar #Sync
stn %o3, [THREAD_REG + T_LOFAULT]
mov ASI_USER, %asi
cmp %o2, SHORTCOPY ! make sure there is enough to align
ble,pt %ncc, .ci_smallest
andcc %o1, 0x7, %o3 ! is dest long word aligned
bnz,pn %ncc, .ci_align
andcc %o1, 1, %o3 ! is dest byte aligned
! Destination is long word aligned
.ci_al_src:
andcc %o0, 7, %o3
brnz,pt %o3, .ci_src_dst_unal8
nop
/*
* Special case for handling when src and dest are both long word aligned
* and total data to move is less than FP_COPY bytes
* Also handles finish up for large block moves, so may be less than 32 bytes
*/
.ci_medlong:
subcc %o2, 31, %o2 ! adjust length to allow cc test
ble,pt %ncc, .ci_medl31
nop
.ci_medl32:
ldxa [%o0]%asi, %o4 ! move 32 bytes
subcc %o2, 32, %o2 ! decrement length count by 32
stx %o4, [%o1]
ldxa [%o0+8]%asi, %o4
stx %o4, [%o1+8]
ldxa [%o0+16]%asi, %o4
add %o0, 32, %o0 ! increase src ptr by 32
stx %o4, [%o1+16]
ldxa [%o0-8]%asi, %o4
add %o1, 32, %o1 ! increase dst ptr by 32
bgu,pt %ncc, .ci_medl32 ! repeat if at least 32 bytes left
stx %o4, [%o1-8]
.ci_medl31:
addcc %o2, 24, %o2 ! adjust count to be off by 7
ble,pt %ncc, .ci_medl7 ! skip if 7 or fewer bytes left
nop
.ci_medl8:
ldxa [%o0]%asi, %o4 ! move 8 bytes
add %o0, 8, %o0 ! increase src ptr by 8
subcc %o2, 8, %o2 ! decrease count by 8
add %o1, 8, %o1 ! increase dst ptr by 8
bgu,pt %ncc, .ci_medl8
stx %o4, [%o1-8]
.ci_medl7:
addcc %o2, 7, %o2 ! finish adjustment of remaining count
bnz,pt %ncc, .ci_small4 ! do final bytes if not finished
nop
.ci_smallx: ! finish up and exit
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT]
.ci_smallqx:
retl
mov %g0, %o0
.ci_small4:
cmp %o2, 4
blt,pt %ncc, .ci_small3x ! skip if less than 4 bytes left
nop !
lda [%o0]%asi, %o4 ! move 4 bytes
add %o0, 4, %o0 ! increase src ptr by 4
add %o1, 4, %o1 ! increase dst ptr by 4
subcc %o2, 4, %o2 ! decrease count by 4
bz %ncc, .ci_smallx
stw %o4, [%o1-4]
.ci_small3x: ! Exactly 1, 2, or 3 bytes remain
subcc %o2, 1, %o2 ! reduce count for cc test
lduba [%o0]%asi, %o4 ! load one byte
bz,pt %ncc, .ci_smallx
stb %o4, [%o1] ! store one byte
lduba [%o0+1]%asi, %o4 ! load second byte
subcc %o2, 1, %o2
bz,pt %ncc, .ci_smallx
stb %o4, [%o1+1] ! store second byte
lduba [%o0+2]%asi, %o4 ! load third byte
ba .ci_smallx
stb %o4, [%o1+2] ! store third byte
.ci_smallest: ! 7 or fewer bytes remain
cmp %o2, 4
blt,pt %ncc, .ci_small3x
nop
lduba [%o0]%asi, %o4 ! read byte
subcc %o2, 4, %o2 ! reduce count by 4
stb %o4, [%o1] ! write byte
lduba [%o0+1]%asi, %o4 ! repeat for total of 4 bytes
add %o0, 4, %o0 ! advance src by 4
stb %o4, [%o1+1]
lduba [%o0-2]%asi, %o4
add %o1, 4, %o1 ! advance dst by 4
stb %o4, [%o1-2]
lduba [%o0-1]%asi, %o4
bnz,pt %ncc, .ci_small3x
stb %o4, [%o1-1]
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT]
retl
mov %g0, %o0
.ci_align:
bnz,pt %ncc, .ci_al_d1
.ci_al_d1f: ! dest is now half word aligned
andcc %o1, 2, %o3 ! is dest word aligned
bnz,pt %ncc, .ci_al_d2
.ci_al_d2f: ! dest is now word aligned
andcc %o1, 4, %o3 ! is dest longword aligned?
bz,pt %ncc, .ci_al_src
nop
.ci_al_d4: ! dest is word aligned; src is unknown
lduba [%o0]%asi, %o4 ! move a word (src align unknown)
lduba [%o0+1]%asi, %o3
sll %o4, 24, %o4 ! position
sll %o3, 16, %o3 ! position
or %o4, %o3, %o3 ! merge
lduba [%o0+2]%asi, %o4
sll %o4, 8, %o4 ! position
or %o4, %o3, %o3 ! merge
lduba [%o0+3]%asi, %o4
or %o4, %o3, %o4 ! merge
stw %o4,[%o1] ! store four bytes
add %o0, 4, %o0 ! adjust src by 4
add %o1, 4, %o1 ! adjust dest by 4
sub %o2, 4, %o2 ! adjust count by 4
andcc %o0, 7, %o3 ! check for src long word alignment
brz,pt %o3, .ci_medlong
.ci_src_dst_unal8:
! dst is 8-byte aligned, src is not
! Size is less than FP_COPY
! Following code is to select for alignment
andcc %o0, 0x3, %o3 ! test word alignment
bz,pt %ncc, .ci_medword
nop
andcc %o0, 0x1, %o3 ! test halfword alignment
bnz,pt %ncc, .ci_med_byte ! go to byte move if not halfword
andcc %o0, 0x2, %o3 ! test which byte alignment
ba .ci_medhalf
nop
.ci_al_d1: ! align dest to half word
lduba [%o0]%asi, %o4 ! move a byte
add %o0, 1, %o0
stb %o4, [%o1]
add %o1, 1, %o1
andcc %o1, 2, %o3 ! is dest word aligned
bz,pt %ncc, .ci_al_d2f
sub %o2, 1, %o2
.ci_al_d2: ! align dest to word
lduba [%o0]%asi, %o4 ! move a half-word (src align unknown)
lduba [%o0+1]%asi, %o3
sll %o4, 8, %o4 ! position
or %o4, %o3, %o4 ! merge
sth %o4, [%o1]
add %o0, 2, %o0
add %o1, 2, %o1
andcc %o1, 4, %o3 ! is dest longword aligned?
bz,pt %ncc, .ci_al_src
sub %o2, 2, %o2
ba .ci_al_d4
nop
/*
* Handle all cases where src and dest are aligned on word
* boundaries. Use unrolled loops for better performance.
* This option wins over standard large data move when
* source and destination is in cache for medium
* to short data moves.
*/
.ci_medword:
subcc %o2, 31, %o2 ! adjust length to allow cc test
ble,pt %ncc, .ci_medw31
nop
.ci_medw32:
lda [%o0]%asi, %o4 ! move a block of 32 bytes
stw %o4, [%o1]
lda [%o0+4]%asi, %o4
stw %o4, [%o1+4]
lda [%o0+8]%asi, %o4
stw %o4, [%o1+8]
lda [%o0+12]%asi, %o4
stw %o4, [%o1+12]
lda [%o0+16]%asi, %o4
stw %o4, [%o1+16]
lda [%o0+20]%asi, %o4
subcc %o2, 32, %o2 ! decrement length count
stw %o4, [%o1+20]
lda [%o0+24]%asi, %o4
add %o0, 32, %o0 ! increase src ptr by 32
stw %o4, [%o1+24]
lda [%o0-4]%asi, %o4
add %o1, 32, %o1 ! increase dst ptr by 32
bgu,pt %ncc, .ci_medw32 ! repeat if at least 32 bytes left
stw %o4, [%o1-4]
.ci_medw31:
addcc %o2, 24, %o2 ! adjust count to be off by 7
ble,pt %ncc, .ci_medw7 ! skip if 7 or fewer bytes left
nop !
.ci_medw15:
lda [%o0]%asi, %o4 ! move a block of 8 bytes
subcc %o2, 8, %o2 ! decrement length count
stw %o4, [%o1]
add %o0, 8, %o0 ! increase src ptr by 8
lda [%o0-4]%asi, %o4
add %o1, 8, %o1 ! increase dst ptr by 8
bgu,pt %ncc, .ci_medw15
stw %o4, [%o1-4]
.ci_medw7:
addcc %o2, 7, %o2 ! finish adjustment of remaining count
bz,pt %ncc, .ci_smallx ! exit if finished
cmp %o2, 4
blt,pt %ncc, .ci_small3x ! skip if less than 4 bytes left
nop !
lda [%o0]%asi, %o4 ! move 4 bytes
add %o0, 4, %o0 ! increase src ptr by 4
add %o1, 4, %o1 ! increase dst ptr by 4
subcc %o2, 4, %o2 ! decrease count by 4
bnz .ci_small3x
stw %o4, [%o1-4]
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT]
retl
mov %g0, %o0
.ci_medhalf:
subcc %o2, 31, %o2 ! adjust length to allow cc test
ble,pt %ncc, .ci_medh31
nop
.ci_medh32: ! load and store block of 32 bytes
subcc %o2, 32, %o2 ! decrement length count
lduha [%o0]%asi, %o4 ! move 32 bytes
lduwa [%o0+2]%asi, %o3
sllx %o4, 48, %o4
sllx %o3, 16, %o3
or %o4, %o3, %o3
lduha [%o0+6]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1]
lduha [%o0+8]%asi, %o4
lduwa [%o0+10]%asi, %o3
sllx %o4, 48, %o4
sllx %o3, 16, %o3
or %o4, %o3, %o3
lduha [%o0+14]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1+8]
lduha [%o0+16]%asi, %o4
lduwa [%o0+18]%asi, %o3
sllx %o4, 48, %o4
sllx %o3, 16, %o3
or %o4, %o3, %o3
lduha [%o0+22]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1+16]
add %o0, 32, %o0 ! increase src ptr by 32
add %o1, 32, %o1 ! increase dst ptr by 32
lduha [%o0-8]%asi, %o4
lduwa [%o0-6]%asi, %o3
sllx %o4, 48, %o4
sllx %o3, 16, %o3
or %o4, %o3, %o3
lduha [%o0-2]%asi, %o4
or %o3, %o4, %o4
bgu,pt %ncc, .ci_medh32 ! repeat if at least 32 bytes left
stx %o4, [%o1-8]
.ci_medh31:
addcc %o2, 24, %o2 ! adjust count to be off by 7
ble,pt %ncc, .ci_medh7 ! skip if 7 or fewer bytes left
nop !
.ci_medh15:
lduha [%o0]%asi, %o4 ! move 16 bytes
subcc %o2, 8, %o2 ! decrement length count
lduwa [%o0+2]%asi, %o3
sllx %o4, 48, %o4
sllx %o3, 16, %o3
or %o4, %o3, %o3
add %o1, 8, %o1 ! increase dst ptr by 8
lduha [%o0+6]%asi, %o4
add %o0, 8, %o0 ! increase src ptr by 8
or %o4, %o3, %o4
bgu,pt %ncc, .ci_medh15
stx %o4, [%o1-8]
.ci_medh7:
addcc %o2, 7, %o2 ! finish adjustment of remaining count
bz,pt %ncc, .ci_smallx ! exit if finished
cmp %o2, 4
blt,pt %ncc, .ci_small3x ! skip if less than 4 bytes left
nop !
lduha [%o0]%asi, %o4
sll %o4, 16, %o4
lduha [%o0+2]%asi, %o3
or %o3, %o4, %o4
subcc %o2, 4, %o2
add %o0, 4, %o0
add %o1, 4, %o1
bnz .ci_small3x
stw %o4, [%o1-4]
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT]
retl
mov %g0, %o0
.align 16
.ci_med_byte:
bnz,pt %ncc, .ci_medbh32a ! go to correct byte move
subcc %o2, 31, %o2 ! adjust length to allow cc test
ble,pt %ncc, .ci_medb31
nop
.ci_medb32: ! Alignment 1 or 5
subcc %o2, 32, %o2 ! decrement length count
lduba [%o0]%asi, %o4 ! load and store a block of 32 bytes
sllx %o4, 56, %o3
lduha [%o0+1]%asi, %o4
sllx %o4, 40, %o4
or %o4, %o3, %o3
lduwa [%o0+3]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+7]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1]
lduba [%o0+8]%asi, %o4
sllx %o4, 56, %o3
lduha [%o0+9]%asi, %o4
sllx %o4, 40, %o4
or %o4, %o3, %o3
lduwa [%o0+11]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+15]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1+8]
lduba [%o0+16]%asi, %o4
sllx %o4, 56, %o3
lduha [%o0+17]%asi, %o4
sllx %o4, 40, %o4
or %o4, %o3, %o3
lduwa [%o0+19]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+23]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1+16]
add %o0, 32, %o0 ! increase src ptr by 32
add %o1, 32, %o1 ! increase dst ptr by 32
lduba [%o0-8]%asi, %o4
sllx %o4, 56, %o3
lduha [%o0-7]%asi, %o4
sllx %o4, 40, %o4
or %o4, %o3, %o3
lduwa [%o0-5]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0-1]%asi, %o4
or %o4, %o3, %o4
bgu,pt %ncc, .ci_medb32 ! repeat if at least 32 bytes left
stx %o4, [%o1-8]
.ci_medb31: ! 31 or fewer bytes remaining
addcc %o2, 24, %o2 ! adjust count to be off by 7
ble,pt %ncc, .ci_medb7 ! skip if 7 or fewer bytes left
nop !
.ci_medb15:
lduba [%o0]%asi, %o4 ! load and store a block of 8 bytes
subcc %o2, 8, %o2 ! decrement length count
sllx %o4, 56, %o3
lduha [%o0+1]%asi, %o4
sllx %o4, 40, %o4
or %o4, %o3, %o3
lduwa [%o0+3]%asi, %o4
add %o1, 8, %o1 ! increase dst ptr by 16
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+7]%asi, %o4
add %o0, 8, %o0 ! increase src ptr by 16
or %o4, %o3, %o4
bgu,pt %ncc, .ci_medb15
stx %o4, [%o1-8]
.ci_medb7:
addcc %o2, 7, %o2 ! finish adjustment of remaining count
bz,pt %ncc, .ci_smallx ! exit if finished
cmp %o2, 4
blt,pt %ncc, .ci_small3x ! skip if less than 4 bytes left
nop !
lduba [%o0]%asi, %o4 ! move 4 bytes
sll %o4, 24, %o3
lduha [%o0+1]%asi, %o4
sll %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+3]%asi, %o4
or %o4, %o3, %o4
subcc %o2, 4, %o2
add %o0, 4, %o0
add %o1, 4, %o1
bnz .ci_small3x
stw %o4, [%o1-4]
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT]
retl
mov %g0, %o0
.align 16
.ci_medbh32a: ! Alignment 3 or 7
ble,pt %ncc, .ci_medbh31
nop
.ci_medbh32: ! Alignment 3 or 7
subcc %o2, 32, %o2 ! decrement length count
lduba [%o0]%asi, %o4 ! load and store a block of 32 bytes
sllx %o4, 56, %o3
lduwa [%o0+1]%asi, %o4
sllx %o4, 24, %o4
or %o4, %o3, %o3
lduha [%o0+5]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+7]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1]
lduba [%o0+8]%asi, %o4
sllx %o4, 56, %o3
lduwa [%o0+9]%asi, %o4
sllx %o4, 24, %o4
or %o4, %o3, %o3
lduha [%o0+13]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+15]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1+8]
lduba [%o0+16]%asi, %o4
sllx %o4, 56, %o3
lduwa [%o0+17]%asi, %o4
sllx %o4, 24, %o4
or %o4, %o3, %o3
lduha [%o0+21]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+23]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1+16]
add %o0, 32, %o0 ! increase src ptr by 32
add %o1, 32, %o1 ! increase dst ptr by 32
lduba [%o0-8]%asi, %o4
sllx %o4, 56, %o3
lduwa [%o0-7]%asi, %o4
sllx %o4, 24, %o4
or %o4, %o3, %o3
lduha [%o0-3]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0-1]%asi, %o4
or %o4, %o3, %o4
bgu,pt %ncc, .ci_medbh32 ! repeat if at least 32 bytes left
stx %o4, [%o1-8]
.ci_medbh31:
addcc %o2, 24, %o2 ! adjust count to be off by 7
ble,pt %ncc, .ci_medb7 ! skip if 7 or fewer bytes left
nop !
.ci_medbh15:
lduba [%o0]%asi, %o4 ! load and store a block of 8 bytes
sllx %o4, 56, %o3
lduwa [%o0+1]%asi, %o4
sllx %o4, 24, %o4
or %o4, %o3, %o3
lduha [%o0+5]%asi, %o4
sllx %o4, 8, %o4
or %o4, %o3, %o3
lduba [%o0+7]%asi, %o4
or %o4, %o3, %o4
stx %o4, [%o1]
subcc %o2, 8, %o2 ! decrement length count
add %o1, 8, %o1 ! increase dst ptr by 8
add %o0, 8, %o0 ! increase src ptr by 8
bgu,pt %ncc, .ci_medbh15
stx %o4, [%o1-8]
ba .ci_medb7
nop
/*
* End of small copy in code (no window)
*
*/
/*
* Long copy in code (using register window and fp regs)
*
*/
.ci_copy_more:
sethi %hi(copyio_fault), %o3
or %o3, %lo(copyio_fault), %o3
membar #Sync
stn %o3, [THREAD_REG + T_LOFAULT]
/*
* Following code is for large copies. We know there is at
* least FP_COPY bytes available. FP regs are used, so
* we save registers and fp regs before starting
*/
save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
or SAVED_LOFAULT, FPUSED_FLAG, SAVED_LOFAULT
rd %fprs, %g1 ! check for unused fp
! if fprs.fef == 0, set it.
! Setting it when already set costs more than checking
andcc %g1, FPRS_FEF, %g1 ! test FEF, fprs.du = fprs.dl = 0
bz,pt %ncc, .ci_fp_unused
mov ASI_USER, %asi
BST_FP_TOSTACK(%o3)
ba .ci_fp_ready
.ci_fp_unused:
prefetcha [%i0 + (1 * CACHE_LINE)]%asi, #one_read
wr %g0, FPRS_FEF, %fprs ! fprs.fef = 1
.ci_fp_ready:
rd %gsr, %l5 ! save %gsr value
andcc %i1, 1, %o3 ! is dest byte aligned
bnz,pt %ncc, .ci_big_d1
.ci_big_d1f: ! dest is now half word aligned
andcc %i1, 2, %o3
bnz,pt %ncc, .ci_big_d2
.ci_big_d2f: ! dest is now word aligned
andcc %i1, 4, %o3
bnz,pt %ncc, .ci_big_d4
.ci_big_d4f: ! dest is long word aligned
andcc %i0, 7, %o3 ! is src long word aligned
brnz,pt %o3, .ci_big_unal8
prefetcha [%i0 + (2 * CACHE_LINE)]%asi, #one_read
! Src and dst are long word aligned
! align dst to 64 byte boundary
andcc %i1, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned
brz,pn %o3, .ci_al_to_64
nop
sub %o3, 64, %o3 ! %o3 has negative bytes to move
add %i2, %o3, %i2 ! adjust remaining count
andcc %o3, 8, %o4 ! odd long words to move?
brz,pt %o4, .ci_al_to_16
nop
add %o3, 8, %o3
ldxa [%i0]%asi, %o4
add %i0, 8, %i0 ! increment src ptr
add %i1, 8, %i1 ! increment dst ptr
stx %o4, [%i1-8]
! Dest is aligned on 16 bytes, src 8 byte aligned
.ci_al_to_16:
andcc %o3, 0x30, %o4 ! pair of long words to move?
brz,pt %o4, .ci_al_to_64
nop
.ci_al_mv_16:
add %o3, 16, %o3
ldxa [%i0]%asi, %o4
stx %o4, [%i1]
add %i0, 16, %i0 ! increment src ptr
ldxa [%i0-8]%asi, %o4
stx %o4, [%i1+8]
andcc %o3, 0x30, %o4
brnz,pt %o4, .ci_al_mv_16
add %i1, 16, %i1 ! increment dst ptr
! Dest is aligned on 64 bytes, src 8 byte aligned
.ci_al_to_64:
! Determine source alignment
! to correct 8 byte offset
andcc %i0, 32, %o3
brnz,pn %o3, .ci_aln_1
andcc %i0, 16, %o3
brnz,pn %o3, .ci_aln_01
andcc %i0, 8, %o3
brz,pn %o3, .ci_aln_000
prefetcha [%i0 + (3 * CACHE_LINE)]%asi, #one_read
ba .ci_aln_001
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
.ci_aln_01:
brnz,pn %o3, .ci_aln_011
prefetcha [%i0 + (3 * CACHE_LINE)]%asi, #one_read
ba .ci_aln_010
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
.ci_aln_1:
andcc %i0, 16, %o3
brnz,pn %o3, .ci_aln_11
andcc %i0, 8, %o3
brnz,pn %o3, .ci_aln_101
prefetcha [%i0 + (3 * CACHE_LINE)]%asi, #one_read
ba .ci_aln_100
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
.ci_aln_11:
brz,pn %o3, .ci_aln_110
prefetcha [%i0 + (3 * CACHE_LINE)]%asi, #one_read
.ci_aln_111:
! Alignment off by 8 bytes
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
ldda [%i0]%asi, %d0
add %i0, 8, %i0
sub %i2, 8, %i2
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_111_loop:
ldda [%i0]ASI_BLK_AIUS,%d16 ! block load
subcc %o3, 64, %o3
fmovd %d16, %d2
fmovd %d18, %d4
fmovd %d20, %d6
fmovd %d22, %d8
fmovd %d24, %d10
fmovd %d26, %d12
fmovd %d28, %d14
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
fmovd %d30, %d0
bgt,pt %ncc, .ci_aln_111_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
std %d0, [%i1]
ba .ci_remain_stuff
add %i1, 8, %i1
! END OF aln_111
.ci_aln_110:
! Alignment off by 16 bytes
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
ldda [%i0]%asi, %d0
ldda [%i0+8]%asi, %d2
add %i0, 16, %i0
sub %i2, 16, %i2
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_110_loop:
ldda [%i0]ASI_BLK_AIUS,%d16 ! block load
subcc %o3, 64, %o3
fmovd %d16, %d4
fmovd %d18, %d6
fmovd %d20, %d8
fmovd %d22, %d10
fmovd %d24, %d12
fmovd %d26, %d14
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
fmovd %d28, %d0
fmovd %d30, %d2
bgt,pt %ncc, .ci_aln_110_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
std %d0, [%i1]
std %d2, [%i1+8]
ba .ci_remain_stuff
add %i1, 16, %i1
! END OF aln_110
.ci_aln_101:
! Alignment off by 24 bytes
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
ldda [%i0]%asi, %d0
ldda [%i0+8]%asi, %d2
ldda [%i0+16]%asi, %d4
add %i0, 24, %i0
sub %i2, 24, %i2
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_101_loop:
ldda [%i0]ASI_BLK_AIUS,%d16 ! block load
subcc %o3, 64, %o3
fmovd %d16, %d6
fmovd %d18, %d8
fmovd %d20, %d10
fmovd %d22, %d12
fmovd %d24, %d14
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
fmovd %d26, %d0
fmovd %d28, %d2
fmovd %d30, %d4
bgt,pt %ncc, .ci_aln_101_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
std %d0, [%i1]
std %d2, [%i1+8]
std %d4, [%i1+16]
ba .ci_remain_stuff
add %i1, 24, %i1
! END OF aln_101
.ci_aln_100:
! Alignment off by 32 bytes
ldda [%i0]%asi, %d0
ldda [%i0+8]%asi, %d2
ldda [%i0+16]%asi,%d4
ldda [%i0+24]%asi,%d6
add %i0, 32, %i0
sub %i2, 32, %i2
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_100_loop:
ldda [%i0]ASI_BLK_AIUS,%d16 ! block load
subcc %o3, 64, %o3
fmovd %d16, %d8
fmovd %d18, %d10
fmovd %d20, %d12
fmovd %d22, %d14
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
fmovd %d24, %d0
fmovd %d26, %d2
fmovd %d28, %d4
fmovd %d30, %d6
bgt,pt %ncc, .ci_aln_100_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
std %d0, [%i1]
std %d2, [%i1+8]
std %d4, [%i1+16]
std %d6, [%i1+24]
ba .ci_remain_stuff
add %i1, 32, %i1
! END OF aln_100
.ci_aln_011:
! Alignment off by 40 bytes
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
ldda [%i0]%asi, %d0
ldda [%i0+8]%asi, %d2
ldda [%i0+16]%asi, %d4
ldda [%i0+24]%asi, %d6
ldda [%i0+32]%asi, %d8
add %i0, 40, %i0
sub %i2, 40, %i2
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_011_loop:
ldda [%i0]ASI_BLK_AIUS,%d16 ! block load
subcc %o3, 64, %o3
fmovd %d16, %d10
fmovd %d18, %d12
fmovd %d20, %d14
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
fmovd %d22, %d0
fmovd %d24, %d2
fmovd %d26, %d4
fmovd %d28, %d6
fmovd %d30, %d8
bgt,pt %ncc, .ci_aln_011_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
std %d0, [%i1]
std %d2, [%i1+8]
std %d4, [%i1+16]
std %d6, [%i1+24]
std %d8, [%i1+32]
ba .ci_remain_stuff
add %i1, 40, %i1
! END OF aln_011
.ci_aln_010:
! Alignment off by 48 bytes
ldda [%i0]%asi, %d0
ldda [%i0+8]%asi, %d2
ldda [%i0+16]%asi, %d4
ldda [%i0+24]%asi, %d6
ldda [%i0+32]%asi, %d8
ldda [%i0+40]%asi, %d10
add %i0, 48, %i0
sub %i2, 48, %i2
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_010_loop:
ldda [%i0]ASI_BLK_AIUS,%d16 ! block load
subcc %o3, 64, %o3
fmovd %d16, %d12
fmovd %d18, %d14
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
fmovd %d20, %d0
fmovd %d22, %d2
fmovd %d24, %d4
fmovd %d26, %d6
fmovd %d28, %d8
fmovd %d30, %d10
bgt,pt %ncc, .ci_aln_010_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
std %d0, [%i1]
std %d2, [%i1+8]
std %d4, [%i1+16]
std %d6, [%i1+24]
std %d8, [%i1+32]
std %d10, [%i1+40]
ba .ci_remain_stuff
add %i1, 48, %i1
! END OF aln_010
.ci_aln_001:
! Alignment off by 56 bytes
ldda [%i0]%asi, %d0
ldda [%i0+8]%asi, %d2
ldda [%i0+16]%asi, %d4
ldda [%i0+24]%asi, %d6
ldda [%i0+32]%asi, %d8
ldda [%i0+40]%asi, %d10
ldda [%i0+48]%asi, %d12
add %i0, 56, %i0
sub %i2, 56, %i2
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_001_loop:
ldda [%i0]ASI_BLK_AIUS,%d16 ! block load
subcc %o3, 64, %o3
fmovd %d16, %d14
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
fmovd %d18, %d0
fmovd %d20, %d2
fmovd %d22, %d4
fmovd %d24, %d6
fmovd %d26, %d8
fmovd %d28, %d10
fmovd %d30, %d12
bgt,pt %ncc, .ci_aln_001_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
std %d0, [%i1]
std %d2, [%i1+8]
std %d4, [%i1+16]
std %d6, [%i1+24]
std %d8, [%i1+32]
std %d10, [%i1+40]
std %d12, [%i1+48]
ba .ci_remain_stuff
add %i1, 56, %i1
! END OF aln_001
.ci_aln_000:
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
andn %i2, 0x7f, %o3 ! %o3 is multiple of 2*block size
and %i2, 0x7f, %i2 ! residue bytes in %i2
sub %i1, %i0, %i1
.ci_aln_000_loop:
ldda [%i0]ASI_BLK_AIUS,%d0
subcc %o3, 64, %o3
stxa %g0,[%i0+%i1]ASI_STBI_P ! block initializing store
stda %d0,[%i0+%i1]ASI_BLK_P
add %i0, 64, %i0
bgt,pt %ncc, .ci_aln_000_loop
prefetcha [%i0 + (4 * CACHE_LINE)]%asi, #one_read
add %i1, %i0, %i1
! END OF aln_000
.ci_remain_stuff:
subcc %i2, 31, %i2 ! adjust length to allow cc test
ble,pt %ncc, .ci_aln_31
nop
.ci_aln_32:
ldxa [%i0]%asi, %o4 ! move 32 bytes
subcc %i2, 32, %i2 ! decrement length count by 32
stx %o4, [%i1]
ldxa [%i0+8]%asi, %o4
stx %o4, [%i1+8]
ldxa [%i0+16]%asi, %o4
add %i0, 32, %i0 ! increase src ptr by 32
stx %o4, [%i1+16]
ldxa [%i0-8]%asi, %o4
add %i1, 32, %i1 ! increase dst ptr by 32
bgu,pt %ncc, .ci_aln_32 ! repeat if at least 32 bytes left
stx %o4, [%i1-8]
.ci_aln_31:
addcc %i2, 24, %i2 ! adjust count to be off by 7
ble,pt %ncc, .ci_aln_7 ! skip if 7 or fewer bytes left
nop !
.ci_aln_15:
ldxa [%i0]%asi, %o4 ! move 8 bytes
add %i0, 8, %i0 ! increase src ptr by 8
subcc %i2, 8, %i2 ! decrease count by 8
add %i1, 8, %i1 ! increase dst ptr by 8
bgu,pt %ncc, .ci_aln_15
stx %o4, [%i1-8] !
.ci_aln_7:
addcc %i2, 7, %i2 ! finish adjustment of remaining count
bz,pt %ncc, .ci_exit ! exit if finished
cmp %i2, 4
blt,pt %ncc, .ci_unaln3x ! skip if less than 4 bytes left
nop !
lda [%i0]%asi, %o4 ! move 4 bytes
add %i0, 4, %i0 ! increase src ptr by 4
add %i1, 4, %i1 ! increase dst ptr by 4
subcc %i2, 4, %i2 ! decrease count by 4
bnz .ci_unaln3x
stw %o4, [%i1-4]
ba .ci_exit
nop
! destination alignment code
.ci_big_d1:
lduba [%i0]%asi, %o4 ! move a byte
add %i0, 1, %i0
stb %o4, [%i1]
add %i1, 1, %i1
andcc %i1, 2, %o3
bz,pt %ncc, .ci_big_d2f
sub %i2, 1, %i2
.ci_big_d2: ! dest is now at least half word aligned
lduba [%i0]%asi, %o4 ! move a half-word (src align unknown)
lduba [%i0+1]%asi, %o3
add %i0, 2, %i0
sll %o4, 8, %o4 ! position
or %o4, %o3, %o4 ! merge
sth %o4, [%i1]
add %i1, 2, %i1
andcc %i1, 4, %o3
bz,pt %ncc, .ci_big_d4f
sub %i2, 2, %i2
.ci_big_d4: ! dest is at least word aligned
nop
lduba [%i0]%asi, %o4 ! move a word (src align unknown)
lduba [%i0+1]%asi, %o3
sll %o4, 24, %o4 ! position
sll %o3, 16, %o3 ! position
or %o4, %o3, %o3 ! merge
lduba [%i0+2]%asi, %o4
sll %o4, 8, %o4 ! position
or %o4, %o3, %o3 ! merge
lduba [%i0+3]%asi, %o4
or %o4, %o3, %o4 ! merge
stw %o4,[%i1] ! store four bytes
add %i0, 4, %i0 ! adjust src by 4
add %i1, 4, %i1 ! adjust dest by 4
ba .ci_big_d4f
sub %i2, 4, %i2 ! adjust count by 4
! Dst is on 8 byte boundary; src is not;
.ci_big_unal8:
andcc %i1, 0x3f, %o3 ! is dst 64-byte block aligned?
bz %ncc, .ci_unalnsrc
sub %o3, 64, %o3 ! %o3 will be multiple of 8
neg %o3 ! bytes until dest is 64 byte aligned
sub %i2, %o3, %i2 ! update cnt with bytes to be moved
! Move bytes according to source alignment
andcc %i0, 0x1, %o4
bnz %ncc, .ci_unalnbyte ! check for byte alignment
nop
andcc %i0, 2, %o4 ! check for half word alignment
bnz %ncc, .ci_unalnhalf
nop
! Src is word aligned, move bytes until dest 64 byte aligned
.ci_unalnword:
lda [%i0]%asi, %o4 ! load 4 bytes
stw %o4, [%i1] ! and store 4 bytes
lda [%i0+4]%asi, %o4 ! load 4 bytes
add %i0, 8, %i0 ! increase src ptr by 8
stw %o4, [%i1+4] ! and store 4 bytes
subcc %o3, 8, %o3 ! decrease count by 8
bnz %ncc, .ci_unalnword
add %i1, 8, %i1 ! increase dst ptr by 8
ba .ci_unalnsrc
nop
! Src is half-word aligned, move bytes until dest 64 byte aligned
.ci_unalnhalf:
lduha [%i0]%asi, %o4 ! load 2 bytes
sllx %o4, 32, %i3 ! shift left
lduwa [%i0+2]%asi, %o4
or %o4, %i3, %i3
sllx %i3, 16, %i3
lduha [%i0+6]%asi, %o4
or %o4, %i3, %i3
stx %i3, [%i1]
add %i0, 8, %i0
subcc %o3, 8, %o3
bnz %ncc, .ci_unalnhalf
add %i1, 8, %i1
ba .ci_unalnsrc
nop
! Src is Byte aligned, move bytes until dest 64 byte aligned
.ci_unalnbyte:
sub %i1, %i0, %i1 ! share pointer advance
.ci_unalnbyte_loop:
lduba [%i0]%asi, %o4
sllx %o4, 56, %i3
lduha [%i0+1]%asi, %o4
sllx %o4, 40, %o4
or %o4, %i3, %i3
lduha [%i0+3]%asi, %o4
sllx %o4, 24, %o4
or %o4, %i3, %i3
lduha [%i0+5]%asi, %o4
sllx %o4, 8, %o4
or %o4, %i3, %i3
lduba [%i0+7]%asi, %o4
or %o4, %i3, %i3
stx %i3, [%i1+%i0]
subcc %o3, 8, %o3
bnz %ncc, .ci_unalnbyte_loop
add %i0, 8, %i0
add %i1,%i0, %i1 ! restore pointer
! Destination is now block (64 byte aligned), src is not 8 byte aligned
.ci_unalnsrc:
andn %i2, 0x3f, %i3 ! %i3 is multiple of block size
and %i2, 0x3f, %i2 ! residue bytes in %i2
!
andn %i0, 0x7, %o4 ! %o4 has long word aligned src address
add %i0, %i3, %i0 ! advance %i0 to after multiple of 8
ldda [%o4]%asi, %d0 ! fetch partial word
.ci_unaln_by8:
ldda [%o4+8]%asi, %d2
add %o4, 8, %o4
faligndata %d0, %d2, %d16
subcc %i3, 8, %i3
std %d16, [%i1]
fmovd %d2, %d0
bgu,pt %ncc, .ci_unaln_by8
add %i1, 8, %i1
.ci_unaln_short:
cmp %i2, 8
blt,pt %ncc, .ci_unalnfin
nop
lduba [%i0]%asi, %o4
sll %o4, 24, %o3
lduba [%i0+1]%asi, %o4
sll %o4, 16, %o4
or %o4, %o3, %o3
lduba [%i0+2]%asi, %o4
sll %o4, 8, %o4
or %o4, %o3, %o3
lduba [%i0+3]%asi, %o4
or %o4, %o3, %o3
stw %o3, [%i1]
lduba [%i0+4]%asi, %o4
sll %o4, 24, %o3
lduba [%i0+5]%asi, %o4
sll %o4, 16, %o4
or %o4, %o3, %o3
lduba [%i0+6]%asi, %o4
sll %o4, 8, %o4
or %o4, %o3, %o3
lduba [%i0+7]%asi, %o4
or %o4, %o3, %o3
stw %o3, [%i1+4]
add %i0, 8, %i0
add %i1, 8, %i1
sub %i2, 8, %i2
.ci_unalnfin:
cmp %i2, 4
blt,pt %ncc, .ci_unalnz
tst %i2
lduba [%i0]%asi, %o3 ! read byte
subcc %i2, 4, %i2 ! reduce count by 4
sll %o3, 24, %o3 ! position
lduba [%i0+1]%asi, %o4
sll %o4, 16, %o4 ! position
or %o4, %o3, %o3 ! merge
lduba [%i0+2]%asi, %o4
sll %o4, 8, %o4 ! position
or %o4, %o3, %o3 ! merge
add %i1, 4, %i1 ! advance dst by 4
lduba [%i0+3]%asi, %o4
add %i0, 4, %i0 ! advance src by 4
or %o4, %o3, %o4 ! merge
bnz,pt %ncc, .ci_unaln3x
stw %o4, [%i1-4]
ba .ci_exit
nop
.ci_unalnz:
bz,pt %ncc, .ci_exit
wr %l5, %g0, %gsr ! restore %gsr
.ci_unaln3x: ! Exactly 1, 2, or 3 bytes remain
subcc %i2, 1, %i2 ! reduce count for cc test
lduba [%i0]%asi, %o4 ! load one byte
bz,pt %ncc, .ci_exit
stb %o4, [%i1] ! store one byte
lduba [%i0+1]%asi, %o4 ! load second byte
subcc %i2, 1, %i2
bz,pt %ncc, .ci_exit
stb %o4, [%i1+1] ! store second byte
lduba [%i0+2]%asi, %o4 ! load third byte
stb %o4, [%i1+2] ! store third byte
.ci_exit:
brnz %g1, .ci_fp_restore
nop
FZERO
wr %g1, %g0, %fprs
ba,pt %ncc, .ci_ex2
membar #Sync
.ci_fp_restore:
BLD_FP_FROMSTACK(%o4)
.ci_ex2:
andn SAVED_LOFAULT, FPUSED_FLAG, SAVED_LOFAULT
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
ret
restore %g0, 0, %o0
.copyin_err:
ldn [THREAD_REG + T_COPYOPS], %o4
brz %o4, 2f
nop
ldn [%o4 + CP_COPYIN], %g2
jmp %g2
nop
2:
retl
mov -1, %o0
#else /* NIAGARA_IMPL */
.do_copyin:
!
! Check the length and bail if zero.
!
tst %o2
bnz,pt %ncc, 1f
nop
retl
clr %o0
1:
sethi %hi(copyio_fault), %o4
or %o4, %lo(copyio_fault), %o4
sethi %hi(copyio_fault_nowindow), %o3
ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT
or %o3, %lo(copyio_fault_nowindow), %o3
membar #Sync
stn %o3, [THREAD_REG + T_LOFAULT]
mov %o0, SAVE_SRC
mov %o1, SAVE_DST
mov %o2, SAVE_COUNT
!
!
!
!
.dcibcp:
!
! pointer. A very fast 4 instruction loop.
!
.align 16
.dcicl:
stb %o4, [%o1 + %o3]
inccc %o3
bl,a,pt %ncc, .dcicl
lduba [%o0 + %o3]ASI_USER, %o4
!
!
!
!
.dci_ns:
!
!
!
! We're single byte aligned.
!
sethi %hi(hw_copy_limit_1), %o3
ld [%o3 + %lo(hw_copy_limit_1)], %o3
!
! Is HW copy on? If not do everything byte for byte.
!
tst %o3
bz,pn %icc, .dcibcp
subcc %o3, %o2, %o3
!
! Are we bigger than the HW limit? If not
! go to byte for byte.
!
bge,pt %ncc, .dcibcp
nop
!
!
.dcih8:
!
!
!
! We're eight byte aligned.
!
sethi %hi(hw_copy_limit_8), %o3
ld [%o3 + %lo(hw_copy_limit_8)], %o3
!
! Is HW assist on? If not, do it with the aligned copy.
!
tst %o3
bz,pn %icc, .dcis8
subcc %o3, %o2, %o3
bge %ncc, .dcis8
nop
ba,pt %ncc, .big_copyin
nop
.dcis8:
!
! Housekeeping for copy loops. Uses same idea as in the byte for
! byte copy loop above.
!
add %o0, %o2, %o0
add %o1, %o2, %o1
sub %g0, %o2, %o3
ba,pt %ncc, .didebc
srl %o2, 3, %o2 ! Number of 8 byte chunks to copy
!
! 4 byte aligned?
!
.dcih4:
bnz %ncc, .dcih2
sethi %hi(hw_copy_limit_4), %o3
ld [%o3 + %lo(hw_copy_limit_4)], %o3
!
! Is HW assist on? If not, do it with the aligned copy.
!
tst %o3
bz,pn %icc, .dcis4
subcc %o3, %o2, %o3
!
!
.dcis4:
!
!
.dcih2:
!
! We're two byte aligned. Check for "smallness"
! done in delay at .dcih4
!
bleu,pt %ncc, .dcis2
sethi %hi(hw_copy_limit_2), %o3
ld [%o3 + %lo(hw_copy_limit_2)], %o3
!
! Is HW assist on? If not, do it with the aligned copy.
!
tst %o3
bz,pn %icc, .dcis2
subcc %o3, %o2, %o3
!
! Are we larger than the HW limit?
!
bge %ncc, .dcis2
nop
!
!
!
!
.dcis2:
!
!
! code.
!
!
!
!
.align 32
.didebc:
!
!
!
!
!
!
.align 32
.didfbc:
!
!
!
!
!
! copy.
!
.align 32
.didtbc:
!
!
!
!
.dcifh:
! We're going off to do a block copy.
! Switch fault hendlers and grab a window. We
! kernel data to this point.
stn %o4, [THREAD_REG + T_LOFAULT]
! Copy in that reach here are larger than 256 bytes. The
! hw_copy_limit_1 is set to 256. Never set this limit less
! 128 bytes.
save %sp, -SA(MINFRAME), %sp
.do_blockcopyin:
mov %i1, %i5
mov %i0, %i1
mov %i5, %i0
! Block (64 bytes) align the destination.
andcc %i0, 0x3f, %i3 ! is dst block aligned
bz %ncc, copyin_blalign ! dst already block aligned
sub %i3, 0x40, %i3
neg %i3 ! bytes till dst 64 bytes aligned
sub %i2, %i3, %i2 ! update i2 with new count
! Based on source and destination alignment do
! either 8 bytes, 4 bytes, 2 bytes or byte copy.
! Is dst & src 8B aligned
or %i0, %i1, %o2
andcc %o2, 0x7, %g0
bz %ncc, .ci_alewdcp
nop
! Is dst & src 4B aligned
andcc %o2, 0x3, %g0
bz %ncc, .ci_alwdcp
nop
! Is dst & src 2B aligned
andcc %o2, 0x1, %g0
bz %ncc, .ci_alhlfwdcp
nop
! 1B aligned
1: lduba [%i1]ASI_USER, %o2
stb %o2, [%i0]
inc %i1
deccc %i3
bgu,pt %ncc, 1b
inc %i0
ba copyin_blalign
nop
! dst & src 4B aligned
.ci_alwdcp:
lda [%i1]ASI_USER, %o2
st %o2, [%i0]
add %i1, 0x4, %i1
subcc %i3, 0x4, %i3
bgu,pt %ncc, .ci_alwdcp
add %i0, 0x4, %i0
ba copyin_blalign
nop
! dst & src 2B aligned
.ci_alhlfwdcp:
lduha [%i1]ASI_USER, %o2
stuh %o2, [%i0]
add %i1, 0x2, %i1
subcc %i3, 0x2, %i3
bgu,pt %ncc, .ci_alhlfwdcp
add %i0, 0x2, %i0
ba copyin_blalign
nop
! dst & src 8B aligned
.ci_alewdcp:
ldxa [%i1]ASI_USER, %o2
stx %o2, [%i0]
add %i1, 0x8, %i1
subcc %i3, 0x8, %i3
bgu,pt %ncc, .ci_alewdcp
add %i0, 0x8, %i0
copyin_blalign:
andn %i2, 0x3f, %i3 ! %i3 count is multiple of block size
sub %i2, %i3, %i2 ! Residue bytes in %i2
mov ASI_BLK_INIT_ST_QUAD_LDD_P, %asi
andcc %i1, 0xf, %o2 ! is src quadword aligned
bz,pn %xcc, .ci_blkcpy ! src offset in %o2 (last 4-bits)
nop
cmp %o2, 0x8
bg .ci_upper_double
nop
bl .ci_lower_double
nop
! Falls through when source offset is equal to 8 i.e.
! source is double word aligned.
sub %i1, %o2, %i1 ! align the src at 16 bytes.
andn %i1, 0x3f, %l0 ! %l0 has block aligned source
prefetcha [%l0]ASI_USER, #one_read
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
add %l0, 0x40, %l0
.ci_loop0:
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l4
prefetcha [%l0]ASI_USER, #one_read
stxa %l3, [%i0+0x0]%asi
stxa %l4, [%i0+0x8]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
stxa %l5, [%i0+0x10]%asi
stxa %l2, [%i0+0x18]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l4
stxa %l3, [%i0+0x20]%asi
stxa %l4, [%i0+0x28]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
stxa %l5, [%i0+0x30]%asi
stxa %l2, [%i0+0x38]%asi
add %l0, 0x40, %l0
subcc %i3, 0x40, %i3
bgu,pt %xcc, .ci_loop0
add %i0, 0x40, %i0
ba .ci_blkdone
add %i1, %o2, %i1 ! increment the source by src offset
! the src offset was stored in %o2
.ci_lower_double:
sub %i1, %o2, %i1 ! align the src at 16 bytes.
sll %o2, 3, %o0 ! %o0 left shift
mov 0x40, %o1
sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
andn %i1, 0x3f, %l0 ! %l0 has block aligned source
prefetcha [%l0]ASI_USER, #one_read
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2 ! partial data in %l2
! and %l3 has complete
! data
add %l0, 0x40, %l0
.ci_loop1:
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l4 ! %l4 has partial data
! for this read.
ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6) ! merge %l2, %l3 and %l4
! into %l2 and %l3
prefetcha [%l0]ASI_USER, #one_read
stxa %l2, [%i0+0x0]%asi
stxa %l3, [%i0+0x8]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6) ! merge %l2 with %l5 and
! %l4 from previous read
! into %l4 and %l5
stxa %l4, [%i0+0x10]%asi
stxa %l5, [%i0+0x18]%asi
! Repeat the same for next 32 bytes.
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l4
ALIGN_DATA(%l2, %l3, %l4, %o0, %o1, %l6)
stxa %l2, [%i0+0x20]%asi
stxa %l3, [%i0+0x28]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
ALIGN_DATA(%l4, %l5, %l2, %o0, %o1, %l6)
stxa %l4, [%i0+0x30]%asi
stxa %l5, [%i0+0x38]%asi
add %l0, 0x40, %l0
subcc %i3, 0x40, %i3
bgu,pt %xcc, .ci_loop1
add %i0, 0x40, %i0
ba .ci_blkdone
add %i1, %o2, %i1 ! increment the source by src offset
! the src offset was stored in %o2
.ci_upper_double:
sub %i1, %o2, %i1 ! align the src at 16 bytes.
sub %o2, 0x8, %o0
sll %o0, 3, %o0 ! %o0 left shift
mov 0x40, %o1
sub %o1, %o0, %o1 ! %o1 right shift = (64 - left shift)
andn %i1, 0x3f, %l0 ! %l0 has block aligned source
prefetcha [%l0]ASI_USER, #one_read
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2 ! partial data in %l3
! for this read and
! no data in %l2
add %l0, 0x40, %l0
.ci_loop2:
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l4 ! %l4 has complete data
! and %l5 has partial
ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6) ! merge %l3, %l4 and %l5
! into %l3 and %l4
prefetcha [%l0]ASI_USER, #one_read
stxa %l3, [%i0+0x0]%asi
stxa %l4, [%i0+0x8]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6) ! merge %l2 and %l3 with
! %l5 from previous read
! into %l5 and %l2
stxa %l5, [%i0+0x10]%asi
stxa %l2, [%i0+0x18]%asi
! Repeat the same for next 32 bytes.
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l4
ALIGN_DATA(%l3, %l4, %l5, %o0, %o1, %l6)
stxa %l3, [%i0+0x20]%asi
stxa %l4, [%i0+0x28]%asi
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
ALIGN_DATA(%l5, %l2, %l3, %o0, %o1, %l6)
stxa %l5, [%i0+0x30]%asi
stxa %l2, [%i0+0x38]%asi
add %l0, 0x40, %l0
subcc %i3, 0x40, %i3
bgu,pt %xcc, .ci_loop2
add %i0, 0x40, %i0
ba .ci_blkdone
add %i1, %o2, %i1 ! increment the source by src offset
! the src offset was stored in %o2
! Do fast copy using ASI_BLK_INIT_ST_QUAD_LDD_P
.ci_blkcpy:
andn %i1, 0x3f, %o0 ! %o0 has block aligned source
prefetcha [%o0]ASI_USER, #one_read
add %o0, 0x40, %o0
1:
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l0
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l2
add %i1, 0x10, %i1
prefetcha [%o0]ASI_USER, #one_read
stxa %l0, [%i0+0x0]%asi
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l4
add %i1, 0x10, %i1
ldda [%i1]ASI_BLK_INIT_QUAD_LDD_AIUS, %l6
add %i1, 0x10, %i1
stxa %l1, [%i0+0x8]%asi
stxa %l2, [%i0+0x10]%asi
stxa %l3, [%i0+0x18]%asi
stxa %l4, [%i0+0x20]%asi
stxa %l5, [%i0+0x28]%asi
stxa %l6, [%i0+0x30]%asi
stxa %l7, [%i0+0x38]%asi
add %o0, 0x40, %o0
subcc %i3, 0x40, %i3
bgu,pt %xcc, 1b
add %i0, 0x40, %i0
.ci_blkdone:
membar #Sync
brz,pt %i2, .copyin_exit
nop
! Handle trailing bytes
cmp %i2, 0x8
blu,pt %ncc, .ci_residue
nop
! Can we do some 8B ops
or %i1, %i0, %o2
andcc %o2, 0x7, %g0
bnz %ncc, .ci_last4
nop
! Do 8byte ops as long as possible
.ci_last8:
ldxa [%i1]ASI_USER, %o2
stx %o2, [%i0]
add %i1, 0x8, %i1
sub %i2, 0x8, %i2
cmp %i2, 0x8
bgu,pt %ncc, .ci_last8
add %i0, 0x8, %i0
brz,pt %i2, .copyin_exit
nop
ba .ci_residue
nop
.ci_last4:
! Can we do 4B ops
andcc %o2, 0x3, %g0
bnz %ncc, .ci_last2
nop
1:
lda [%i1]ASI_USER, %o2
st %o2, [%i0]
add %i1, 0x4, %i1
sub %i2, 0x4, %i2
cmp %i2, 0x4
bgu,pt %ncc, 1b
add %i0, 0x4, %i0
brz,pt %i2, .copyin_exit
nop
ba .ci_residue
nop
.ci_last2:
! Can we do 2B ops
andcc %o2, 0x1, %g0
bnz %ncc, .ci_residue
nop
1:
lduha [%i1]ASI_USER, %o2
stuh %o2, [%i0]
add %i1, 0x2, %i1
sub %i2, 0x2, %i2
cmp %i2, 0x2
bgu,pt %ncc, 1b
add %i0, 0x2, %i0
brz,pt %i2, .copyin_exit
nop
! Copy the residue as byte copy
.ci_residue:
lduba [%i1]ASI_USER, %i4
stb %i4, [%i0]
inc %i1
deccc %i2
bgu,pt %xcc, .ci_residue
inc %i0
.copyin_exit:
membar #Sync
stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
ret
restore %g0, 0, %o0
.copyin_err:
ldn [THREAD_REG + T_COPYOPS], %o4
brz %o4, 2f
nop
ldn [%o4 + CP_COPYIN], %g2
jmp %g2
nop
2:
retl
mov -1, %o0
#endif /* NIAGARA_IMPL */
SET_SIZE(copyin)
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
xcopyin(const void *uaddr, void *kaddr, size_t count)
{ return (0); }
#else /* lint */
ENTRY(xcopyin)
sethi %hi(.xcopyin_err), REAL_LOFAULT
b .do_copyin
or REAL_LOFAULT, %lo(.xcopyin_err), REAL_LOFAULT
.xcopyin_err:
ldn [THREAD_REG + T_COPYOPS], %o4
brz %o4, 2f
nop
ldn [%o4 + CP_XCOPYIN], %g2
jmp %g2
nop
2:
retl
mov %g1, %o0
SET_SIZE(xcopyin)
#endif /* lint */
#ifdef lint
/*ARGSUSED*/
int
xcopyin_little(const void *uaddr, void *kaddr, size_t count)
{ return (0); }
#else /* lint */
ENTRY(xcopyin_little)
sethi %hi(.little_err), %o4
ldn [THREAD_REG + T_LOFAULT], %o5
or %o4, %lo(.little_err), %o4
membar #Sync ! sync error barrier
stn %o4, [THREAD_REG + T_LOFAULT]
subcc %g0, %o2, %o3
add %o0, %o2, %o0
bz,pn %ncc, 2f ! check for zero bytes
sub %o2, 1, %o4
add %o1, %o2, %o1
lduba [%o0+%o3]ASI_AIUSL, %o4
1: stb %o4, [%o1+%o3]
inccc %o3
sub %o0, 2, %o0 ! get next byte
bcc,a,pt %ncc, 1b
lduba [%o0+%o3]ASI_AIUSL, %o4
2: membar #Sync ! sync error barrier
stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
retl
mov %g0, %o0 ! return (0)
.little_err:
membar #Sync ! sync error barrier
stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
retl
mov %g1, %o0
SET_SIZE(xcopyin_little)
#endif /* lint */
/*
* Copy a block of storage - must not overlap (from + len <= to).
* No fault handler installed (to be called under on_fault())
*/
#if defined(lint)
/* ARGSUSED */
void
copyin_noerr(const void *ufrom, void *kto, size_t count)
{}
#else /* lint */
ENTRY(copyin_noerr)
sethi %hi(.copyio_noerr), REAL_LOFAULT
b .do_copyin
or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
.copyio_noerr:
jmp SAVED_LOFAULT
nop
SET_SIZE(copyin_noerr)
#endif /* lint */
/*
* Copy a block of storage - must not overlap (from + len <= to).
* No fault handler installed (to be called under on_fault())
*/
#if defined(lint)
/* ARGSUSED */
void
copyout_noerr(const void *kfrom, void *uto, size_t count)
{}
#else /* lint */
ENTRY(copyout_noerr)
sethi %hi(.copyio_noerr), REAL_LOFAULT
b .do_copyout
or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
SET_SIZE(copyout_noerr)
#endif /* lint */
#if defined(lint)
int use_hw_bcopy = 1;
int use_hw_bzero = 1;
uint_t hw_copy_limit_1 = 0x100;
uint_t hw_copy_limit_2 = 0x200;
uint_t hw_copy_limit_4 = 0x400;
uint_t hw_copy_limit_8 = 0x400;
#else /* !lint */
.align 4
DGDEF(use_hw_bcopy)
.word 1
DGDEF(use_hw_bzero)
.word 1
DGDEF(hw_copy_limit_1)
.word 0x100
DGDEF(hw_copy_limit_2)
.word 0x200
DGDEF(hw_copy_limit_4)
.word 0x400
DGDEF(hw_copy_limit_8)
.word 0x400
.align 64
.section ".text"
#endif /* !lint */
/*
* hwblkclr - clears block-aligned, block-multiple-sized regions that are
*/
#ifdef lint
/*ARGSUSED*/
int
{
return(0);
}
#else /* lint */
2:
3:
#endif /* lint */
#ifdef lint
/* Copy 32 bytes of data from src to dst using physical addresses */
/*ARGSUSED*/
void
{}
#else /*!lint */
/*
* Copy 32 bytes of data from src (%o0) to dst (%o1)
* using physical addresses.
*/
#endif /* lint */
/*
* Zero a block of storage.
*
* uzero is used by the kernel to zero a block in user address space.
*/
/*
*
* For fewer than 7 bytes stores, bytes will be zeroed.
*
* For less than 15 bytes stores, align the address on 4 byte boundary.
* Then store as many 4-byte chunks, followed by trailing bytes.
*
* For sizes greater than 15 bytes, align the address on 8 byte boundary.
* if (count > 128) {
* store as many 8-bytes chunks to block align the address
* store using ASI_BLK_INIT_QUAD_LDD_AIUS (uzero)
* }
* Store as many 8-byte chunks, followed by trailing bytes.
*/
#if defined(lint)
/* ARGSUSED */
int
{ return(0); }
/* ARGSUSED */
void
{}
#else /* lint */
!
!
!
!
/*
* We got here because of a fault during kzero or if
* uzero or bzero was called with t_lofault non-zero.
* Otherwise we've already run screaming from the room.
* Errno value is in %g1. Note that we're here iff
* we did set t_lofault.
*/
.zeroerr:
!
!
!
!
1:
2:
!
!
retl ! return
3:
!
! We're here because %o5 was non-zero. It was non-zero
! because either LOFAULT_SET was present, a previous fault
! handler was present or both. In all cases we need to reset
! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
! before we either simply return the error or we invoke the
! previously specified handler.
!
be %ncc, 2b
stn %o5, [THREAD_REG + T_LOFAULT]
jmp %o5 ! goto real handler
nop
SET_SIZE(kzero)
SET_SIZE(uzero)
#endif /* lint */
/*
* Zero a block of storage.
*/
#if defined(lint)
/* ARGSUSED */
void
bzero(void *addr, size_t count)
{}
#else /* lint */
ENTRY(bzero)
wr %g0, ASI_P, %asi
ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector
tst %o5
bz,pt %ncc, .do_zero
sethi %hi(.zeroerr), %o2
or %o2, %lo(.zeroerr), %o2
membar #Sync ! sync error barrier
stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector
.do_zero:
cmp %o1, 7
blu,pn %ncc, .byteclr
nop
cmp %o1, 15
blu,pn %ncc, .wdalign
nop
andcc %o0, 7, %o3 ! is add aligned on a 8 byte bound
bz,pt %ncc, .blkalign ! already double aligned
sub %o3, 8, %o3 ! -(bytes till double aligned)
add %o1, %o3, %o1 ! update o1 with new count
1:
stba %g0, [%o0]%asi
inccc %o3
bl,pt %ncc, 1b
inc %o0
! Now address is double aligned
.blkalign:
cmp %o1, 0x80 ! check if there are 128 bytes to set
blu,pn %ncc, .bzero_small
mov %o1, %o3
sethi %hi(use_hw_bzero), %o2
ld [%o2 + %lo(use_hw_bzero)], %o2
tst %o2
bz %ncc, .bzero_small
mov %o1, %o3
rd %asi, %o3
wr %g0, ASI_BLK_INIT_ST_QUAD_LDD_P, %asi
cmp %o3, ASI_P
bne,a %ncc, .algnblk
wr %g0, ASI_BLK_INIT_QUAD_LDD_AIUS, %asi
.algnblk:
andcc %o0, 0x3f, %o3 ! is block aligned?
bz,pt %ncc, .bzero_blk
sub %o3, 0x40, %o3 ! -(bytes till block aligned)
add %o1, %o3, %o1 ! o1 is the remainder
! Clear -(%o3) bytes till block aligned
1:
stxa %g0, [%o0]%asi
addcc %o3, 8, %o3
bl,pt %ncc, 1b
add %o0, 8, %o0
.bzero_blk:
and %o1, 0x3f, %o3 ! calc bytes left after blk clear
andn %o1, 0x3f, %o4 ! calc size of blocks in bytes
cmp %o4, 0x100 ! 256 bytes or more
blu,pn %ncc, 3f
nop
2:
stxa %g0, [%o0+0x0]%asi
stxa %g0, [%o0+0x40]%asi
stxa %g0, [%o0+0x80]%asi
stxa %g0, [%o0+0xc0]%asi
stxa %g0, [%o0+0x8]%asi
stxa %g0, [%o0+0x10]%asi
stxa %g0, [%o0+0x18]%asi
stxa %g0, [%o0+0x20]%asi
stxa %g0, [%o0+0x28]%asi
stxa %g0, [%o0+0x30]%asi
stxa %g0, [%o0+0x38]%asi
stxa %g0, [%o0+0x48]%asi
stxa %g0, [%o0+0x50]%asi
stxa %g0, [%o0+0x58]%asi
stxa %g0, [%o0+0x60]%asi
stxa %g0, [%o0+0x68]%asi
stxa %g0, [%o0+0x70]%asi
stxa %g0, [%o0+0x78]%asi
stxa %g0, [%o0+0x88]%asi
stxa %g0, [%o0+0x90]%asi
stxa %g0, [%o0+0x98]%asi
stxa %g0, [%o0+0xa0]%asi
stxa %g0, [%o0+0xa8]%asi
stxa %g0, [%o0+0xb0]%asi
stxa %g0, [%o0+0xb8]%asi
stxa %g0, [%o0+0xc8]%asi
stxa %g0, [%o0+0xd0]%asi
stxa %g0, [%o0+0xd8]%asi
stxa %g0, [%o0+0xe0]%asi
stxa %g0, [%o0+0xe8]%asi
stxa %g0, [%o0+0xf0]%asi
stxa %g0, [%o0+0xf8]%asi
sub %o4, 0x100, %o4
cmp %o4, 0x100
bgu,pt %ncc, 2b
add %o0, 0x100, %o0
3:
! ... check if 64 bytes to set
cmp %o4, 0x40
blu %ncc, .bzero_blk_done
nop
4:
stxa %g0, [%o0+0x0]%asi
stxa %g0, [%o0+0x8]%asi
stxa %g0, [%o0+0x10]%asi
stxa %g0, [%o0+0x18]%asi
stxa %g0, [%o0+0x20]%asi
stxa %g0, [%o0+0x28]%asi
stxa %g0, [%o0+0x30]%asi
stxa %g0, [%o0+0x38]%asi
subcc %o4, 0x40, %o4
bgu,pt %ncc, 3b
add %o0, 0x40, %o0
.bzero_blk_done:
membar #Sync
!
! Undo asi register setting.
!
rd %asi, %o4
wr %g0, ASI_P, %asi
cmp %o4, ASI_BLK_INIT_ST_QUAD_LDD_P
bne,a %ncc, .bzero_small
wr %g0, ASI_USER, %asi
.bzero_small:
! Set the remaining doubles
subcc %o3, 8, %o3 ! Can we store any doubles?
blu,pn %ncc, .byteclr
and %o1, 7, %o1 ! calc bytes left after doubles
.dbclr:
stxa %g0, [%o0]%asi ! Clear the doubles
subcc %o3, 8, %o3
bgeu,pt %ncc, .dbclr
add %o0, 8, %o0
ba .byteclr
nop
.wdalign:
andcc %o0, 3, %o3 ! is add aligned on a word boundary
bz,pn %ncc, .wdclr
andn %o1, 3, %o3 ! create word sized count in %o3
dec %o1 ! decrement count
stba %g0, [%o0]%asi ! clear a byte
ba .wdalign
inc %o0 ! next byte
.wdclr:
sta %g0, [%o0]%asi ! 4-byte clearing loop
subcc %o3, 4, %o3
bnz,pt %ncc, .wdclr
inc 4, %o0
and %o1, 3, %o1 ! leftover count, if any
.byteclr:
! Set the leftover bytes
brz %o1, .bzero_exit
nop
7:
deccc %o1 ! byte clearing loop
stba %g0, [%o0]%asi
bgu,pt %ncc, 7b
inc %o0
.bzero_exit:
!
!
1:
#endif /* lint */