copy.s revision 9acbbeaf2a1ffe5c14b244867d427714fab43c5c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
/* All Rights Reserved */
/* Copyright (c) 1987, 1988 Microsoft Corporation */
/* All Rights Reserved */
#pragma ident "%Z%%M% %I% %E% SMI"
#include <sys/asm_linkage.h>
#if defined(__lint)
#else /* __lint */
#include "assym.h"
#endif /* __lint */
/*
* Non-temopral access (NTA) alignment requirement
*/
/*
* Copy a block of storage, returning an error code if `from' or
* `to' takes a kernel pagefault which cannot be resolved.
* Returns errno value on pagefault error, 0 if all ok
*/
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
#ifdef DEBUG
jb 0f
jnb 1f
1:
#endif
/*
* pass lofault value as 4th argument to do_copy_fault
*/
/*
* A fault during do_copy_fault is indicated through an errno value
* in %rax and we iretq from the trap handler to here.
*/
#define ARG_FROM 8
#define ARG_TO 12
#define ARG_COUNT 16
#ifdef DEBUG
jb 0f
jnb 1f
0: pushl $.kcopy_panic_msg
#endif
/*
* A fault during do_copy_fault is indicated through an errno value
* in %eax and we iret from the trap handler to here.
*/
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/*
* Copy a block of storage. Similar to kcopy but uses non-temporal
* instructions.
*/
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/* Copy 16 bytes per loop. Uses %rax and %r8 */
#ifdef DEBUG
jb 0f
jnb 1f
1:
#endif
/*
* pass lofault value as 4th argument to do_copy_fault
*/
/*
* Make sure cnt is >= KCOPY_MIN_SIZE
*/
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
/*
* COPY_LOOP_BODY uses %rax and %r8
*/
jnz 2b
#define ARG_FROM 8
#define ARG_TO 12
#define ARG_COUNT 16
/*
* kcopy_nta is not implemented for 32-bit as no performance
* improvement was shown. We simply jump directly to kcopy
* and discard the 4 arguments.
*/
/* COPY_LOOP_BODY needs to use %esi */
jnz 1b
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
void
{}
#else /* __lint */
#if defined(__amd64)
#ifdef DEBUG
jz 1f
jb 0f
jnb 1f
1:
#endif
#ifdef DEBUG
/*
* Setup frame on the run-time stack. The end of the input argument
* area must be aligned on a 16 byte boundary. The stack pointer %rsp,
* always points to the end of the latest allocated stack frame.
* panic(const char *format, ...) is a varargs function. When a
* function taking variable arguments is called, %rax must be set
* to eight times the number of floating point parameters passed
* to the function in SSE registers.
*/
#endif
#define ARG_FROM 4
#define ARG_TO 8
#define ARG_COUNT 12
#ifdef DEBUG
jz 1f
jb 0f
jnb 1f
1:
#endif
#endif /* __i386 */
#endif /* __lint */
/*
* Zero a block of storage, returning an error code if we
* take a kernel pagefault which cannot be resolved.
* Returns errno value on pagefault error, 0 if all ok
*/
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
#ifdef DEBUG
jnb 0f
0:
#endif
/*
* pass lofault value as 3rd argument to do_zero_fault
*/
sstoq /* %rcx = words to clear (%rax=0) */
sstob /* %rcx = residual bytes to clear */
/*
* A fault during do_zero_fault is indicated through an errno value
* in %rax when we iretq to here.
*/
#define ARG_ADDR 8
#define ARG_COUNT 12
#ifdef DEBUG
jnb 0f
#endif
sstol /* %ecx contains words to clear (%eax=0) */
sstob /* %ecx contains residual bytes to clear */
/*
* A fault during do_zero_fault is indicated through an errno value
* in %eax when we iret to here.
*/
#endif /* __i386 */
#endif /* __lint */
/*
* Zero a block of storage.
*/
#if defined(__lint)
/* ARGSUSED */
void
{}
#else /* __lint */
#if defined(__amd64)
#ifdef DEBUG
jnb 0f
0:
#endif
sstoq /* %rcx = words to clear (%rax=0) */
sstob /* %rcx = residual bytes to clear */
#define ARG_ADDR 4
#define ARG_COUNT 8
#ifdef DEBUG
jnb 0f
0:
#endif
#endif /* __i386 */
#endif /* __lint */
/*
* Transfer data to and from user space -
* Note that these routines can cause faults
* It is assumed that the kernel has nothing at
* less than KERNELBASE in the virtual address space.
*
* Note that copyin(9F) and copyout(9F) are part of the
*
* Sigh.
*
* So there's two extremely similar routines - xcopyin_nta() and
* xcopyout_nta() which return the errno that we've faithfully computed.
* This allows other callers (e.g. uiomove(9F)) to work correctly.
* Given that these are used pretty heavily, we expand the calling
* sequences inline for all flavours (rather than making wrappers).
*/
/*
* Copy user data to kernel space.
*/
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* lint */
#if defined(__amd64)
/*
* save args in case we trap and need to rerun as a copyop
*/
#ifdef DEBUG
jnb 1f
1:
#endif
/*
* pass lofault value as 4th argument to do_copy_fault
*/
jmp 3f
3:
jz 2f
/*
* reload args for the copyop
*/
#define ARG_UADDR 4
#define ARG_KADDR 8
#ifdef DEBUG
jnb 1f
1:
#endif
jmp 3f
3:
jz 2f
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/*
* save args in case we trap and need to rerun as a copyop
* %rcx is consumed in this routine so we don't need to save
* it.
*/
#ifdef DEBUG
jnb 1f
1:
#endif
jae 4f
/*
* pass lofault value as 4th argument to do_copy_fault
*/
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
4:
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %rax and we iret from the
* trap handler to here.
*/
3:
jz 2f
/*
* reload args for the copyop
*/
2: leave
#define ARG_UADDR 4
#define ARG_KADDR 8
#define ARG_COUNT 12
#define ARG_CACHED 16
jae 4f
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
4:
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %eax and we iret from the
* trap handler to here.
*/
3:
jz 2f
/* AMD Software Optimization Guide - Section 6.2 */
#endif /* __i386 */
#endif /* __lint */
/*
* Copy kernel data to user space.
*/
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/*
* save args in case we trap and need to rerun as a copyop
*/
#ifdef DEBUG
jnb 1f
1:
#endif
/*
* pass lofault value as 4th argument to do_copy_fault
*/
jmp 3f
3:
jz 2f
/*
* reload args for the copyop
*/
#define ARG_KADDR 4
#define ARG_UADDR 8
#ifdef DEBUG
jnb 1f
1:
#endif
jmp 3f
3:
jz 2f
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/*
* save args in case we trap and need to rerun as a copyop
*/
#ifdef DEBUG
jnb 1f
1:
#endif
jae 4f
/*
* pass lofault value as 4th argument to do_copy_fault
*/
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
4:
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %rax and we iret from the
* trap handler to here.
*/
3:
jz 2f
/*
* reload args for the copyop
*/
2: leave
#define ARG_KADDR 4
#define ARG_UADDR 8
#define ARG_COUNT 12
#define ARG_CACHED 16
jae 4f
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
4:
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %eax and we iret from the
* trap handler to here.
*/
3:
jz 2f
/* AMD Software Optimization Guide - Section 6.2 */
#endif /* __i386 */
#endif /* __lint */
/*
* Copy a null terminated string from one point to another in
* the kernel address space.
*/
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
#ifdef DEBUG
jb 0f
jnb 1f
1:
#endif
/* 5th argument to do_copystr */
#define ARG_FROM 8
#define ARG_TO 12
#define ARG_MAXLEN 16
#define ARG_LENCOPIED 20
#ifdef DEBUG
jb 0f
jnb 1f
0: pushl $.copystr_panic_msg
#endif
/* get the current lofault address */
#endif /* __i386 */
#endif /* __lint */
/*
* Copy a null terminated string from the user address space into
* the kernel address space.
*/
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/*
* save args in case we trap and need to rerun as a copyop
*/
#ifdef DEBUG
jnb 1f
1:
#endif
/*
* pass lofault value as 5th argument to do_copystr
*/
jmp 3f
3:
jz 2f
/*
* reload args for the copyop
*/
#define ARG_UADDR 4
#define ARG_KADDR 8
#ifdef DEBUG
jnb 1f
1:
#endif
jmp 3f
3:
jz 2f
#endif /* __i386 */
#endif /* __lint */
/*
* Copy a null terminated string from the kernel
* address space to the user address space.
*/
#if defined(__lint)
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/*
* save args in case we trap and need to rerun as a copyop
*/
#ifdef DEBUG
jnb 1f
1:
#endif
/*
* pass lofault value as 5th argument to do_copystr
*/
jmp 3f
3:
jz 2f
/*
* reload args for the copyop
*/
#define ARG_KADDR 4
#define ARG_UADDR 8
#ifdef DEBUG
jnb 1f
1:
#endif
jmp 3f
3:
jz 2f
#endif /* __i386 */
#endif /* __lint */
/*
* Since all of the fuword() variants are so similar, we have a macro to spit
* them out. This allows us to create DTrace-unobservable functions easily.
*/
#if defined(__lint)
#if defined(__amd64)
/* ARGSUSED */
int
{ return (0); }
#endif
/* ARGSUSED */
int
{ return (0); }
/* ARGSUSED */
int
{ return (0); }
/* ARGSUSED */
int
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/*
* (Note that we don't save and reload the arguments here
* because their values are not altered in the copy path)
*/
jae 1f; \
ret; \
1: \
jz 2f; \
2: \
ret; \
jae 1f; \
ret; \
1: \
jz 2f; \
2: \
ret; \
#endif /* __i386 */
#endif /* __lint */
/*
* Set user word.
*/
#if defined(__lint)
#if defined(__amd64)
/* ARGSUSED */
int
{ return (0); }
#endif
/* ARGSUSED */
int
{ return (0); }
/* ARGSUSED */
int
{ return (0); }
/* ARGSUSED */
int
{ return (0); }
#else /* lint */
#if defined(__amd64)
/*
* (Note that we don't save and reload the arguments here
* because their values are not altered in the copy path)
*/
jae 1f; \
ret; \
1: \
jz 3f; \
3: \
ret; \
jae 1f; \
ret; \
1: \
jz 3f; \
3: \
ret; \
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
#if defined(__amd64)
/*ARGSUSED*/
void
{}
#endif
/*ARGSUSED*/
void
{}
/*ARGSUSED*/
void
{}
/*ARGSUSED*/
void
{}
#else /* __lint */
#if defined(__amd64)
ret; \
jb 1f; \
ret; \
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
#if defined(__amd64)
/*ARGSUSED*/
void
{}
#endif
/*ARGSUSED*/
void
{}
/*ARGSUSED*/
void
{}
/*ARGSUSED*/
void
{}
#else /* lint */
#if defined(__amd64)
ret; \
jb 1f; \
1: \
ret; \
#endif /* __i386 */
#endif /* lint */
#if defined(__lint)
/*ARGSUSED*/
int
{ return (0); }
/*ARGSUSED*/
void
{}
/*ARGSUSED*/
int
{ return (0); }
/*ARGSUSED*/
void
{}
/*ARGSUSED*/
int
{ return (0); }
/*ARGSUSED*/
void
{}
#else
#if defined(__amd64)
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/*
* Copy a block of storage - must not overlap (from + len <= to).
* No fault handler installed (to be called under on_fault())
*/
/* ARGSUSED */
void
{}
/* ARGSUSED */
void
{}
/*
* Zero a block of storage in user space
*/
/* ARGSUSED */
void
{}
/*
* copy a block of storage in user space
*/
/* ARGSUSED */
void
{}
/*
* copy a string in user space
*/
/* ARGSUSED */
void
{}
#else /* __lint */
#if defined(__amd64)
#ifdef DEBUG
jae 1f
1:
#endif
#ifdef DEBUG
jae 1f
1:
#endif
/* do_copystr expects lofault address in %r8 */
#ifdef DEBUG
jae 1f
1:
#endif
#ifdef DEBUG
jae 1f
1:
#endif
jb 1f
1:
jb 1f
1:
jb 2f
2:
/* do_copystr expects the lofault address in %eax */
#endif /* __i386 */
#ifdef DEBUG
.data
.string "kcopy: arguments below kernelbase"
.string "bcopy: arguments below kernelbase"
.string "kzero: arguments below kernelbase"
.string "bzero: arguments below kernelbase"
.string "copyin: kaddr argument below kernelbase"
.string "xcopyin: kaddr argument below kernelbase"
.string "copyout: kaddr argument below kernelbase"
.string "xcopyout: kaddr argument below kernelbase"
.string "copystr: arguments in user space"
.string "copyinstr: kaddr argument not in kernel address space"
.string "copyoutstr: kaddr argument not in kernel address space"
.string "copyin_noerr: argument not in kernel address space"
.string "copyout_noerr: argument not in kernel address space"
#endif
#endif /* __lint */