asm.h revision e64031e20c39650a7bc902a3e1aba613b9415dee
/** @file
* IPRT - Assembly Functions.
*/
/*
* Copyright (C) 2006-2007 Oracle Corporation
*
* This file is part of VirtualBox Open Source Edition (OSE), as
* available from http://www.virtualbox.org. This file is free software;
* General Public License (GPL) as published by the Free Software
* Foundation, in version 2 as it comes in the "COPYING" file of the
* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
*
* The contents of this file may alternatively be used under the terms
* of the Common Development and Distribution License Version 1.0
* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
* VirtualBox OSE distribution, in which case the provisions of the
* CDDL are applicable instead of those of the GPL.
*
* You may elect to license modified versions of this file under the
* terms and conditions of either the GPL or the CDDL or both.
*/
#ifndef ___iprt_asm_h
#define ___iprt_asm_h
/** @def RT_INLINE_ASM_USES_INTRIN
* Defined as 1 if we're using a _MSC_VER 1400.
* Otherwise defined as 0.
*/
/* Solaris 10 header ugliness */
#ifdef u
#undef u
#endif
#ifdef _MSC_VER
# if _MSC_VER >= 1400
# define RT_INLINE_ASM_USES_INTRIN 1
# include <intrin.h>
/* Emit the intrinsics at all optimization levels. */
# pragma intrinsic(_ReadWriteBarrier)
# pragma intrinsic(__writemsr)
# pragma intrinsic(__outbytestring)
# pragma intrinsic(__outwordstring)
# pragma intrinsic(__outdword)
# pragma intrinsic(__outdwordstring)
# pragma intrinsic(__inbytestring)
# pragma intrinsic(__inwordstring)
# pragma intrinsic(__indwordstring)
# pragma intrinsic(__writecr0)
# pragma intrinsic(__writecr3)
# pragma intrinsic(__writecr4)
# pragma intrinsic(_BitScanForward)
# pragma intrinsic(_BitScanReverse)
# pragma intrinsic(_bittestandset)
# pragma intrinsic(_bittestandreset)
# pragma intrinsic(_bittestandcomplement)
# pragma intrinsic(_byteswap_ushort)
# pragma intrinsic(_byteswap_ulong)
# pragma intrinsic(_interlockedbittestandset)
# pragma intrinsic(_interlockedbittestandreset)
# pragma intrinsic(_InterlockedAnd)
# pragma intrinsic(_InterlockedOr)
# pragma intrinsic(_InterlockedIncrement)
# pragma intrinsic(_InterlockedDecrement)
# pragma intrinsic(_InterlockedExchange)
# pragma intrinsic(_InterlockedExchangeAdd)
# pragma intrinsic(_InterlockedCompareExchange)
# pragma intrinsic(_InterlockedCompareExchange64)
# ifdef RT_ARCH_AMD64
# pragma intrinsic(_mm_mfence)
# pragma intrinsic(_mm_sfence)
# pragma intrinsic(_mm_lfence)
# pragma intrinsic(__writecr8)
# pragma intrinsic(_byteswap_uint64)
# pragma intrinsic(_InterlockedExchange64)
# endif
# endif
#endif
#ifndef RT_INLINE_ASM_USES_INTRIN
# define RT_INLINE_ASM_USES_INTRIN 0
#endif
/** @defgroup grp_asm ASM - Assembly Routines
* @ingroup grp_rt
*
* @remarks The difference between ordered and unordered atomic operations are that
* the former will complete outstanding reads and writes before continuing
* while the latter doesn't make any promisses about the order. Ordered
* operations doesn't, it seems, make any 100% promise wrt to whether
* the operation will complete before any subsequent memory access.
* (please, correct if wrong.)
*
* ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
* are unordered (note the Uo).
*
* @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
* or even optimize assembler instructions away. For instance, in the following code
* the second rdmsr instruction is optimized away because gcc treats that instruction
* as deterministic:
*
* @code
* static inline uint64_t rdmsr_low(int idx)
* {
* uint32_t low;
* __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
* }
* ...
* uint32_t msr1 = rdmsr_low(1);
* foo(msr1);
* msr1 = rdmsr_low(1);
* bar(msr1);
* @endcode
*
* The input parameter of rdmsr_low is the same for both calls and therefore gcc will
* use the result of the first call as input parameter for bar() as well. For rdmsr this
* is not acceptable as this instruction is _not_ deterministic. This applies to reading
* machine status information in general.
*
* @{
*/
/** @def RT_INLINE_ASM_GCC_4_3_X_X86
* Used to work around some 4.3.x register allocation issues in this version of
* the compiler. So far this workaround is still required for 4.4 and 4.5. */
#ifdef __GNUC__
#endif
#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
# define RT_INLINE_ASM_GCC_4_3_X_X86 0
#endif
/** @def RT_INLINE_DONT_USE_CMPXCHG8B
* i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5493) screws up
* RTSemRWRequestWrite semsemrw-lockless-generic.cpp in release builds. PIC
* mode, x86.
*
* Some gcc 4.3.x versions may have register allocation issues with cmpxchg8b
* when in PIC mode on x86.
*/
# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC \
&& defined(RT_ARCH_X86) \
&& ( RT_INLINE_ASM_GCC_4_3_X_X86 \
|| defined(RT_OS_DARWIN)) )
#endif
/** @def RT_INLINE_ASM_EXTERNAL
* Defined as 1 if the compiler does not support inline assembly.
* The ASM* functions will then be implemented in an external .asm file.
*
* @remark At the present time it's unconfirmed whether or not Microsoft skipped
* inline assembly in their AMD64 compiler.
*/
#if defined(_MSC_VER) && defined(RT_ARCH_AMD64)
# define RT_INLINE_ASM_EXTERNAL 1
#else
# define RT_INLINE_ASM_EXTERNAL 0
#endif
/** @def RT_INLINE_ASM_GNU_STYLE
* Defined as 1 if the compiler understands GNU style inline assembly.
*/
#if defined(_MSC_VER)
# define RT_INLINE_ASM_GNU_STYLE 0
#else
# define RT_INLINE_ASM_GNU_STYLE 1
#endif
/** @todo find a more proper place for this structure? */
#pragma pack(1)
/** IDTR */
typedef struct RTIDTR
{
/** Size of the IDT. */
/** Address of the IDT. */
#pragma pack()
#pragma pack(1)
/** GDTR */
typedef struct RTGDTR
{
/** Size of the GDT. */
/** Address of the GDT. */
#pragma pack()
/** @def ASMReturnAddress
* Gets the return address of the current (or calling if you like) function or method.
*/
#ifdef _MSC_VER
# ifdef __cplusplus
extern "C"
# endif
void * _ReturnAddress(void);
# pragma intrinsic(_ReturnAddress)
# define ASMReturnAddress() _ReturnAddress()
# define ASMReturnAddress() __builtin_return_address(0)
#else
# error "Unsupported compiler."
#endif
/**
* Gets the content of the IDTR CPU register.
* @param pIdtr Where to store the IDTR contents.
*/
#else
{
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Sets the content of the IDTR CPU register.
* @param pIdtr Where to load the IDTR contents from
*/
#else
{
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Gets the content of the GDTR CPU register.
* @param pGdtr Where to store the GDTR contents.
*/
#else
{
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Get the cs register.
* @returns cs.
*/
#else
{
# else
{
}
# endif
return SelCS;
}
#endif
/**
* Get the DS register.
* @returns DS.
*/
#else
{
# else
{
}
# endif
return SelDS;
}
#endif
/**
* Get the ES register.
* @returns ES.
*/
#else
{
# else
{
}
# endif
return SelES;
}
#endif
/**
* Get the FS register.
* @returns FS.
*/
#else
{
# else
{
}
# endif
return SelFS;
}
# endif
/**
* Get the GS register.
* @returns GS.
*/
#else
{
# else
{
}
# endif
return SelGS;
}
#endif
/**
* Get the SS register.
* @returns SS.
*/
#else
{
# else
{
}
# endif
return SelSS;
}
#endif
/**
* Get the TR register.
* @returns TR.
*/
#else
{
# else
{
}
# endif
return SelTR;
}
#endif
/**
* Get the [RE]FLAGS register.
* @returns [RE]FLAGS.
*/
#else
{
# ifdef RT_ARCH_AMD64
"popq %0\n\t"
: "=r" (uFlags));
# else
"popl %0\n\t"
: "=r" (uFlags));
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uFlags;
}
#endif
/**
* Set the [RE]FLAGS register.
* @param uFlags The new [RE]FLAGS value.
*/
#else
{
# ifdef RT_ARCH_AMD64
"popfq\n\t"
: : "g" (uFlags));
# else
"popfl\n\t"
: : "g" (uFlags));
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Gets the content of the CPU timestamp counter register.
*
* @returns TSC.
*/
#else
{
RTUINT64U u;
# else
u.u = __rdtsc();
# else
{
}
# endif
# endif
return u.u;
}
#endif
/**
* Performs the cpuid instruction returning all registers.
*
* @param uOperator CPUID operation (eax).
* @param pvEAX Where to store eax.
* @param pvEBX Where to store ebx.
* @param pvECX Where to store ecx.
* @param pvEDX Where to store edx.
* @remark We're using void pointers to ease the use of special bitfield structures and such.
*/
#else
{
# ifdef RT_ARCH_AMD64
__asm__ ("cpuid\n\t"
: "=a" (uRAX),
"=b" (uRBX),
"=c" (uRCX),
"=d" (uRDX)
: "0" (uOperator));
# else
__asm__ ("xchgl %%ebx, %1\n\t"
"cpuid\n\t"
"xchgl %%ebx, %1\n\t"
: "0" (uOperator));
# endif
int aInfo[4];
# else
{
}
# endif
}
#endif
/**
* Performs the cpuid instruction returning all registers.
* Some subfunctions of cpuid take ECX as additional parameter (currently known for EAX=4)
*
* @param uOperator CPUID operation (eax).
* @param uIdxECX ecx index
* @param pvEAX Where to store eax.
* @param pvEBX Where to store ebx.
* @param pvECX Where to store ecx.
* @param pvEDX Where to store edx.
* @remark We're using void pointers to ease the use of special bitfield structures and such.
*/
DECLASM(void) ASMCpuId_Idx_ECX(uint32_t uOperator, uint32_t uIdxECX, void *pvEAX, void *pvEBX, void *pvECX, void *pvEDX);
#else
DECLINLINE(void) ASMCpuId_Idx_ECX(uint32_t uOperator, uint32_t uIdxECX, void *pvEAX, void *pvEBX, void *pvECX, void *pvEDX)
{
# ifdef RT_ARCH_AMD64
__asm__ ("cpuid\n\t"
: "=a" (uRAX),
"=b" (uRBX),
"=c" (uRCX),
"=d" (uRDX)
: "0" (uOperator),
"2" (uIdxECX));
# else
__asm__ ("xchgl %%ebx, %1\n\t"
"cpuid\n\t"
"xchgl %%ebx, %1\n\t"
: "0" (uOperator),
"2" (uIdxECX));
# endif
int aInfo[4];
/* ??? another intrinsic ??? */
# else
{
}
# endif
}
#endif
/**
* Performs the cpuid instruction returning ecx and edx.
*
* @param uOperator CPUID operation (eax).
* @param pvECX Where to store ecx.
* @param pvEDX Where to store edx.
* @remark We're using void pointers to ease the use of special bitfield structures and such.
*/
#else
{
}
#endif
/**
* Performs the cpuid instruction returning edx.
*
* @param uOperator CPUID operation (eax).
* @returns EDX after cpuid operation.
*/
#else
{
# ifdef RT_ARCH_AMD64
__asm__ ("cpuid"
: "=a" (uSpill),
"=d" (xDX)
: "0" (uOperator)
: "rbx", "rcx");
__asm__ ("push %%ebx\n\t"
"cpuid\n\t"
"pop %%ebx\n\t"
: "=a" (uOperator),
"=d" (xDX)
: "0" (uOperator)
: "ecx");
# else
__asm__ ("cpuid"
: "=a" (uOperator),
"=d" (xDX)
: "0" (uOperator)
: "ebx", "ecx");
# endif
int aInfo[4];
# else
{
}
# endif
}
#endif
/**
* Performs the cpuid instruction returning ecx.
*
* @param uOperator CPUID operation (eax).
* @returns ECX after cpuid operation.
*/
#else
{
# ifdef RT_ARCH_AMD64
__asm__ ("cpuid"
: "=a" (uSpill),
"=c" (xCX)
: "0" (uOperator)
: "rbx", "rdx");
__asm__ ("push %%ebx\n\t"
"cpuid\n\t"
"pop %%ebx\n\t"
: "=a" (uOperator),
"=c" (xCX)
: "0" (uOperator)
: "edx");
# else
__asm__ ("cpuid"
: "=a" (uOperator),
"=c" (xCX)
: "0" (uOperator)
: "ebx", "edx");
# endif
int aInfo[4];
# else
{
}
# endif
}
#endif
/**
* Checks if the current CPU supports CPUID.
*
* @returns true if CPUID is supported.
*/
DECLINLINE(bool) ASMHasCpuId(void)
{
#ifdef RT_ARCH_AMD64
return true; /* ASSUME that all amd64 compatible CPUs have cpuid. */
#else /* !RT_ARCH_AMD64 */
bool fRet = false;
__asm__ ("pushf\n\t"
"pop %1\n\t"
"mov %1, %2\n\t"
"xorl $0x200000, %1\n\t"
"push %1\n\t"
"popf\n\t"
"pushf\n\t"
"pop %1\n\t"
"cmpl %1, %2\n\t"
"setne %0\n\t"
"push %2\n\t"
"popf\n\t"
# else
{
}
# endif
return fRet;
#endif /* !RT_ARCH_AMD64 */
}
/**
* Gets the APIC ID of the current CPU.
*
* @returns the APIC ID.
*/
#else
{
# ifdef RT_ARCH_AMD64
__asm__ ("cpuid"
: "=a" (uSpill),
"=b" (xBX)
: "0" (1)
: "rcx", "rdx");
__asm__ ("mov %%ebx,%1\n\t"
"cpuid\n\t"
"xchgl %%ebx,%1\n\t"
: "=a" (uSpill),
"=r" (xBX)
: "0" (1)
: "ecx", "edx");
# else
__asm__ ("cpuid"
: "=a" (uSpill),
"=b" (xBX)
: "0" (1)
: "ecx", "edx");
# endif
int aInfo[4];
# else
{
}
# endif
}
#endif
/**
* Tests if it a genuine Intel CPU based on the ASMCpuId(0) output.
*
* @param uEBX EBX return from ASMCpuId(0)
* @param uECX ECX return from ASMCpuId(0)
* @param uEDX EDX return from ASMCpuId(0)
*/
{
}
/**
* Tests if this is a genuine Intel CPU.
*
* @remarks ASSUMES that cpuid is supported by the CPU.
*/
DECLINLINE(bool) ASMIsIntelCpu(void)
{
}
/**
* Tests if it a authentic AMD CPU based on the ASMCpuId(0) output.
*
* @param uEBX EBX return from ASMCpuId(0)
* @param uECX ECX return from ASMCpuId(0)
* @param uEDX EDX return from ASMCpuId(0)
*/
{
}
/**
* Tests if this is an authentic AMD CPU.
*
* @remarks ASSUMES that cpuid is supported by the CPU.
*/
DECLINLINE(bool) ASMIsAmdCpu(void)
{
}
/**
* Extracts the CPU family from ASMCpuId(1) or ASMCpuId(0x80000001)
*
* @returns Family.
* @param uEAX EAX return from ASMCpuId(1) or ASMCpuId(0x80000001).
*/
{
}
/**
* Extracts the CPU model from ASMCpuId(1) or ASMCpuId(0x80000001), Intel variant.
*
* @returns Model.
* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
*/
{
}
/**
* Extracts the CPU model from ASMCpuId(1) or ASMCpuId(0x80000001), AMD variant.
*
* @returns Model.
* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
*/
{
}
/**
* Extracts the CPU model from ASMCpuId(1) or ASMCpuId(0x80000001)
*
* @returns Model.
* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
* @param fIntel Whether it's an intel CPU. Use ASMIsIntelCpuEx() or ASMIsIntelCpu().
*/
{
}
/**
* Extracts the CPU stepping from ASMCpuId(1) or ASMCpuId(0x80000001)
*
* @returns Model.
* @param uEAX EAX from ASMCpuId(1) or ASMCpuId(0x80000001).
*/
{
return uEAX & 0xf;
}
/**
* Get cr0.
* @returns cr0.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uCR0;
}
#endif
/**
* Sets the CR0 register.
* @param uCR0 The new CR0 value.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Get cr2.
* @returns cr2.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uCR2;
}
#endif
/**
* Sets the CR2 register.
* @param uCR2 The new CR0 value.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Get cr3.
* @returns cr3.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uCR3;
}
#endif
/**
* Sets the CR3 register.
*
* @param uCR3 New CR3 value.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Reloads the CR3 register.
*/
DECLASM(void) ASMReloadCR3(void);
#else
DECLINLINE(void) ASMReloadCR3(void)
{
__writecr3(__readcr3());
RTCCUINTREG u;
# ifdef RT_ARCH_AMD64
"movq %0, %%cr3\n\t"
: "=r" (u));
# else
"movl %0, %%cr3\n\t"
: "=r" (u));
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Get cr4.
* @returns cr4.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
/*mov eax, cr4*/
_emit 0x0f
_emit 0x20
_emit 0xe0
# endif
}
# endif
return uCR4;
}
#endif
/**
* Sets the CR4 register.
*
* @param uCR4 New CR4 value.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
_emit 0x0F
_emit 0x22
# endif
}
# endif
}
#endif
/**
* Get cr8.
* @returns cr8.
* @remark The lock prefix hack for access from non-64-bit modes is NOT used and 0 is returned.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
{
}
# endif
return uCR8;
# else /* !RT_ARCH_AMD64 */
return 0;
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Enables interrupts (EFLAGS.IF).
*/
DECLASM(void) ASMIntEnable(void);
#else
DECLINLINE(void) ASMIntEnable(void)
{
__asm("sti\n");
_enable();
# else
# endif
}
#endif
/**
* Disables interrupts (!EFLAGS.IF).
*/
DECLASM(void) ASMIntDisable(void);
#else
DECLINLINE(void) ASMIntDisable(void)
{
__asm("cli\n");
_disable();
# else
# endif
}
#endif
/**
* Disables interrupts and returns previous xFLAGS.
*/
#else
{
# ifdef RT_ARCH_AMD64
"cli\n\t"
"popq %0\n\t"
: "=r" (xFlags));
# else
"cli\n\t"
"popl %0\n\t"
: "=r" (xFlags));
# endif
xFlags = ASMGetFlags();
_disable();
# else
__asm {
}
# endif
return xFlags;
}
#endif
/**
* Are interrupts enabled?
*
* @returns true / false.
*/
{
}
/**
* Halts the CPU until interrupted.
*/
#else
DECLINLINE(void) ASMHalt(void)
{
# else
__asm {
}
# endif
}
#endif
/**
* The PAUSE variant of NOP for helping hyperthreaded CPUs detecing spin locks.
*/
DECLASM(void) ASMNopPause(void);
#else
DECLINLINE(void) ASMNopPause(void)
{
# else
__asm {
_emit 090h
}
# endif
}
#endif
/**
* Reads a machine specific register.
*
* @returns Register content.
* @param uRegister Register to read.
*/
#else
{
RTUINT64U u;
: "=a" (u.s.Lo),
"=d" (u.s.Hi)
: "c" (uRegister));
# else
{
}
# endif
return u.u;
}
#endif
/**
* Writes a machine specific register.
*
* @returns Register content.
* @param uRegister Register to write to.
* @param u64Val Value to write.
*/
#else
{
RTUINT64U u;
u.u = u64Val;
::"a" (u.s.Lo),
"d" (u.s.Hi),
"c" (uRegister));
__writemsr(uRegister, u.u);
# else
{
}
# endif
}
#endif
/**
* Reads low part of a machine specific register.
*
* @returns Register content.
* @param uRegister Register to read.
*/
#else
{
: "=a" (u32)
: "c" (uRegister)
: "edx");
#else
{
}
# endif
return u32;
}
#endif
/**
* Reads high part of a machine specific register.
*
* @returns Register content.
* @param uRegister Register to read.
*/
#else
{
: "=d" (u32)
: "c" (uRegister)
: "eax");
# else
{
}
# endif
return u32;
}
#endif
/**
* Gets dr0.
*
* @returns dr0.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uDR0;
}
#endif
/**
* Gets dr1.
*
* @returns dr1.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uDR1;
}
#endif
/**
* Gets dr2.
*
* @returns dr2.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uDR2;
}
#endif
/**
* Gets dr3.
*
* @returns dr3.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uDR3;
}
#endif
/**
* Gets dr6.
*
* @returns dr6.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uDR6;
}
#endif
/**
* Reads and clears DR6.
*
* @returns DR6.
*/
#else
{
# ifdef RT_ARCH_AMD64
"movq %1, %%dr6\n\t"
: "=r" (uDR6)
: "r" (uNewValue));
# else
"movl %1, %%dr6\n\t"
: "=r" (uDR6)
: "r" (uNewValue));
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uDR6;
}
#endif
/**
* Gets dr7.
*
* @returns dr7.
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return uDR7;
}
#endif
/**
* Sets dr0.
*
* @param uDRVal Debug register value to write
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Sets dr1.
*
* @param uDRVal Debug register value to write
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Sets dr2.
*
* @param uDRVal Debug register value to write
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Sets dr3.
*
* @param uDRVal Debug register value to write
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Sets dr6.
*
* @param uDRVal Debug register value to write
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Sets dr7.
*
* @param uDRVal Debug register value to write
*/
#else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Compiler memory barrier.
*
* values or any outstanding writes when returning from this function.
*
* This function must be used if non-volatile data is modified by a
* device or the VMM. Typical cases are port access, MMIO access,
* trapping instruction, etc.
*/
# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
DECLINLINE(void) ASMCompilerBarrier(void)
{
{
}
}
#endif
/**
* Writes a 8-bit unsigned integer to an I/O port, ordered.
*
* @param Port I/O port to write to.
* @param u8 8-bit integer to write.
*/
#else
{
:: "Nd" (Port),
"a" (u8));
# else
{
}
# endif
}
#endif
/**
* Reads a 8-bit unsigned integer from an I/O port, ordered.
*
* @returns 8-bit integer.
* @param Port I/O port to read from.
*/
#else
{
: "=a" (u8)
: "Nd" (Port));
# else
{
}
# endif
return u8;
}
#endif
/**
* Writes a 16-bit unsigned integer to an I/O port, ordered.
*
* @param Port I/O port to write to.
* @param u16 16-bit integer to write.
*/
#else
{
:: "Nd" (Port),
"a" (u16));
# else
{
}
# endif
}
#endif
/**
* Reads a 16-bit unsigned integer from an I/O port, ordered.
*
* @returns 16-bit integer.
* @param Port I/O port to read from.
*/
#else
{
: "=a" (u16)
: "Nd" (Port));
# else
{
}
# endif
return u16;
}
#endif
/**
* Writes a 32-bit unsigned integer to an I/O port, ordered.
*
* @param Port I/O port to write to.
* @param u32 32-bit integer to write.
*/
#else
{
:: "Nd" (Port),
"a" (u32));
# else
{
}
# endif
}
#endif
/**
* Reads a 32-bit unsigned integer from an I/O port, ordered.
*
* @returns 32-bit integer.
* @param Port I/O port to read from.
*/
#else
{
: "=a" (u32)
: "Nd" (Port));
# else
{
}
# endif
return u32;
}
#endif
/**
* Writes a string of 8-bit unsigned integer items to an I/O port, ordered.
*
* @param Port I/O port to write to.
* @param pau8 Pointer to the string buffer.
* @param c The number of items to write.
*/
#else
{
: "+S" (pau8),
"+c" (c)
: "d" (Port));
# else
{
}
# endif
}
#endif
/**
* Reads a string of 8-bit unsigned integer items from an I/O port, ordered.
*
* @param Port I/O port to read from.
* @param pau8 Pointer to the string buffer (output).
* @param c The number of items to read.
*/
#else
{
: "+D" (pau8),
"+c" (c)
: "d" (Port));
# else
{
}
# endif
}
#endif
/**
* Writes a string of 16-bit unsigned integer items to an I/O port, ordered.
*
* @param Port I/O port to write to.
* @param pau16 Pointer to the string buffer.
* @param c The number of items to write.
*/
#else
{
: "+S" (pau16),
"+c" (c)
: "d" (Port));
# else
{
}
# endif
}
#endif
/**
* Reads a string of 16-bit unsigned integer items from an I/O port, ordered.
*
* @param Port I/O port to read from.
* @param pau16 Pointer to the string buffer (output).
* @param c The number of items to read.
*/
#else
{
: "+D" (pau16),
"+c" (c)
: "d" (Port));
# else
{
}
# endif
}
#endif
/**
* Writes a string of 32-bit unsigned integer items to an I/O port, ordered.
*
* @param Port I/O port to write to.
* @param pau32 Pointer to the string buffer.
* @param c The number of items to write.
*/
#else
{
: "+S" (pau32),
"+c" (c)
: "d" (Port));
# else
{
}
# endif
}
#endif
/**
* Reads a string of 32-bit unsigned integer items from an I/O port, ordered.
*
* @param Port I/O port to read from.
* @param pau32 Pointer to the string buffer (output).
* @param c The number of items to read.
*/
#else
{
: "+D" (pau32),
"+c" (c)
: "d" (Port));
# else
{
}
# endif
}
#endif
/**
* Atomically Exchange an unsigned 8-bit value, ordered.
*
* @returns Current *pu8 value
* @param pu8 Pointer to the 8-bit variable to update.
* @param u8 The 8-bit value to assign to *pu8.
*/
#else
{
: "=m" (*pu8),
: "1" (u8),
"m" (*pu8));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return u8;
}
#endif
/**
* Atomically Exchange a signed 8-bit value, ordered.
*
* @returns Current *pu8 value
* @param pi8 Pointer to the 8-bit variable to update.
* @param i8 The 8-bit value to assign to *pi8.
*/
{
}
/**
* Atomically Exchange a bool value, ordered.
*
* @returns Current *pf value
* @param pf Pointer to the 8-bit variable to update.
* @param f The 8-bit value to assign to *pi8.
*/
{
#ifdef _MSC_VER
#else
#endif
}
/**
* Atomically Exchange an unsigned 16-bit value, ordered.
*
* @returns Current *pu16 value
* @param pu16 Pointer to the 16-bit variable to update.
* @param u16 The 16-bit value to assign to *pu16.
*/
#else
{
: "=m" (*pu16),
"=r" (u16)
: "1" (u16),
"m" (*pu16));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return u16;
}
#endif
/**
* Atomically Exchange a signed 16-bit value, ordered.
*
* @returns Current *pu16 value
* @param pi16 Pointer to the 16-bit variable to update.
* @param i16 The 16-bit value to assign to *pi16.
*/
{
}
/**
* Atomically Exchange an unsigned 32-bit value, ordered.
*
* @returns Current *pu32 value
* @param pu32 Pointer to the 32-bit variable to update.
* @param u32 The 32-bit value to assign to *pu32.
*/
#else
{
: "=m" (*pu32),
"=r" (u32)
: "1" (u32),
"m" (*pu32));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return u32;
}
#endif
/**
* Atomically Exchange a signed 32-bit value, ordered.
*
* @returns Current *pu32 value
* @param pi32 Pointer to the 32-bit variable to update.
* @param i32 The 32-bit value to assign to *pi32.
*/
{
}
/**
* Atomically Exchange an unsigned 64-bit value, ordered.
*
* @returns Current *pu64 value
* @param pu64 Pointer to the 64-bit variable to update.
* @param u64 The 64-bit value to assign to *pu64.
*/
#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
#else
{
# if defined(RT_ARCH_AMD64)
: "=m" (*pu64),
"=r" (u64)
: "1" (u64),
"m" (*pu64));
# else
{
}
# endif
# else /* !RT_ARCH_AMD64 */
"xchgl %%ebx, %3\n\t"
"1:\n\t"
"lock; cmpxchg8b (%5)\n\t"
"jnz 1b\n\t"
"movl %3, %%ebx\n\t"
/*"xchgl %%esi, %5\n\t"*/
: "=A" (u64),
"=m" (*pu64)
: "0" (*pu64),
"m" ( u32EBX ),
"S" (pu64));
# else /* !PIC */
"lock; cmpxchg8b %1\n\t"
"jnz 1b\n\t"
: "=A" (u64),
"=m" (*pu64)
: "0" (*pu64),
# endif
# else
{
}
# endif
# endif /* !RT_ARCH_AMD64 */
return u64;
}
#endif
/**
* Atomically Exchange an signed 64-bit value, ordered.
*
* @returns Current *pi64 value
* @param pi64 Pointer to the 64-bit variable to update.
* @param i64 The 64-bit value to assign to *pi64.
*/
{
}
/**
* Atomically Exchange a pointer value, ordered.
*
* @returns Current *ppv value
* @param ppv Pointer to the pointer variable to update.
* @param pv The pointer value to assign to *ppv.
*/
{
#if ARCH_BITS == 32
#else
# error "ARCH_BITS is bogus"
#endif
}
/**
* Atomically Exchange a raw-mode context pointer value, ordered.
*
* @returns Current *ppv value
* @param ppvRC Pointer to the pointer variable to update.
* @param pvRC The pointer value to assign to *ppv.
*/
{
}
/**
* Atomically Exchange a ring-0 pointer value, ordered.
*
* @returns Current *ppv value
* @param ppvR0 Pointer to the pointer variable to update.
* @param pvR0 The pointer value to assign to *ppv.
*/
{
#if R0_ARCH_BITS == 32
#else
# error "R0_ARCH_BITS is bogus"
#endif
}
/**
* Atomically Exchange a ring-3 pointer value, ordered.
*
* @returns Current *ppv value
* @param ppvR3 Pointer to the pointer variable to update.
* @param pvR3 The pointer value to assign to *ppv.
*/
{
#if R3_ARCH_BITS == 32
#else
# error "R3_ARCH_BITS is bogus"
#endif
}
/** @def ASMAtomicXchgHandle
* Atomically Exchange a typical IPRT handle value, ordered.
*
* @param ph Pointer to the value to update.
* @param hNew The new value to assigned to *pu.
* @param phRes Where to store the current *ph value.
*
* @remarks This doesn't currently work for all handles (like RTFILE).
*/
#if HC_ARCH_BITS == 32
do { \
} while (0)
do { \
} while (0)
#else
#endif
/**
* Atomically Exchange a value which size might differ
* between platforms or compilers, ordered.
*
* @param pu Pointer to the variable to update.
* @param uNew The value to assign to *pu.
* @todo This is busted as its missing the result argument.
*/
do { \
switch (sizeof(*(pu))) { \
} \
} while (0)
/**
* Atomically Exchange a value which size might differ
* between platforms or compilers, ordered.
*
* @param pu Pointer to the variable to update.
* @param uNew The value to assign to *pu.
* @param puRes Where to store the current *pu value.
*/
do { \
switch (sizeof(*(pu))) { \
case 1: *(uint8_t *)(puRes) = ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
case 2: *(uint16_t *)(puRes) = ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
case 4: *(uint32_t *)(puRes) = ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
case 8: *(uint64_t *)(puRes) = ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
} \
} while (0)
/**
* Atomically Compare and Exchange an unsigned 8-bit value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pu8 Pointer to the value to update.
* @param u8New The new value to assigned to *pu8.
* @param u8Old The old value to *pu8 compare with.
*/
#else
{
"setz %1\n\t"
: "=m" (*pu8),
"=qm" (u8Ret),
"=a" (u8Old)
: "q" (u8New),
"2" (u8Old),
"m" (*pu8));
return (bool)u8Ret;
}
#endif
/**
* Atomically Compare and Exchange a signed 8-bit value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pi8 Pointer to the value to update.
* @param i8New The new value to assigned to *pi8.
* @param i8Old The old value to *pi8 compare with.
*/
{
}
/**
* Atomically Compare and Exchange a bool value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pf Pointer to the value to update.
* @param fNew The new value to assigned to *pf.
* @param fOld The old value to *pf compare with.
*/
{
}
/**
* Atomically Compare and Exchange an unsigned 32-bit value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pu32 Pointer to the value to update.
* @param u32New The new value to assigned to *pu32.
* @param u32Old The old value to *pu32 compare with.
*/
DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
#else
DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
{
"setz %1\n\t"
: "=m" (*pu32),
"=qm" (u8Ret),
"=a" (u32Old)
: "r" (u32New),
"2" (u32Old),
"m" (*pu32));
return (bool)u8Ret;
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# ifdef RT_ARCH_AMD64
# else
# endif
}
return !!u32Ret;
# endif
}
#endif
/**
* Atomically Compare and Exchange a signed 32-bit value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pi32 Pointer to the value to update.
* @param i32New The new value to assigned to *pi32.
* @param i32Old The old value to *pi32 compare with.
*/
DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
{
}
/**
* Atomically Compare and exchange an unsigned 64-bit value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pu64 Pointer to the 64-bit variable to update.
* @param u64New The 64-bit value to assign to *pu64.
* @param u64Old The value to compare with.
*/
#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
#else
{
# elif defined(RT_ARCH_AMD64)
"setz %1\n\t"
: "=m" (*pu64),
"=qm" (u8Ret),
"=a" (u64Old)
: "r" (u64New),
"2" (u64Old),
"m" (*pu64));
return (bool)u8Ret;
# else
bool fRet;
{
}
return fRet;
# endif
# else /* !RT_ARCH_AMD64 */
"lock; cmpxchg8b (%6)\n\t"
"setz %%al\n\t"
"movl %4, %%ebx\n\t"
"movzbl %%al, %%eax\n\t"
: "=a" (u32Ret),
"=d" (u32Spill),
"+m" (*pu64)
# else
"=m" (*pu64)
# endif
: "A" (u64Old),
"m" ( u32EBX ),
"S" (pu64));
# else /* !PIC */
"setz %%al\n\t"
"movzbl %%al, %%eax\n\t"
: "=a" (u32Ret),
"=d" (u32Spill),
"+m" (*pu64)
: "A" (u64Old),
# endif
return (bool)u32Ret;
# else
{
}
return !!u32Ret;
# endif
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Atomically Compare and exchange a signed 64-bit value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pi64 Pointer to the 64-bit variable to update.
* @param i64 The 64-bit value to assign to *pu64.
* @param i64Old The value to compare with.
*/
DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
{
}
/**
* Atomically Compare and Exchange a pointer value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param ppv Pointer to the value to update.
* @param pvNew The new value to assigned to *ppv.
* @param pvOld The old value to *ppv compare with.
*/
{
#if ARCH_BITS == 32
#else
# error "ARCH_BITS is bogus"
#endif
}
/** @def ASMAtomicCmpXchgHandle
* Atomically Compare and Exchange a typical IPRT handle value, ordered.
*
* @param ph Pointer to the value to update.
* @param hNew The new value to assigned to *pu.
* @param hOld The old value to *pu compare with.
* @param fRc Where to store the result.
*
* @remarks This doesn't currently work for all handles (like RTFILE).
*/
#if HC_ARCH_BITS == 32
do { \
(fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
} while (0)
do { \
(fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
} while (0)
#else
#endif
/** @def ASMAtomicCmpXchgSize
* Atomically Compare and Exchange a value which size might differ
* between platforms or compilers, ordered.
*
* @param pu Pointer to the value to update.
* @param uNew The new value to assigned to *pu.
* @param uOld The old value to *pu compare with.
* @param fRc Where to store the result.
*/
do { \
switch (sizeof(*(pu))) { \
case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
break; \
case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
break; \
(fRc) = false; \
break; \
} \
} while (0)
/**
* Atomically Compare and Exchange an unsigned 32-bit value, additionally
* passes back old value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pu32 Pointer to the value to update.
* @param u32New The new value to assigned to *pu32.
* @param u32Old The old value to *pu32 compare with.
* @param pu32Old Pointer store the old value at.
*/
DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old);
#else
DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old)
{
"setz %1\n\t"
: "=m" (*pu32),
"=qm" (u8Ret),
"=a" (*pu32Old)
: "r" (u32New),
"a" (u32Old),
"m" (*pu32));
return (bool)u8Ret;
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# ifdef RT_ARCH_AMD64
# else
# endif
}
return !!u32Ret;
# endif
}
#endif
/**
* Atomically Compare and Exchange a signed 32-bit value, additionally
* passes back old value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pi32 Pointer to the value to update.
* @param i32New The new value to assigned to *pi32.
* @param i32Old The old value to *pi32 compare with.
* @param pi32Old Pointer store the old value at.
*/
DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old, int32_t *pi32Old)
{
return ASMAtomicCmpXchgExU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t *)pi32Old);
}
/**
* Atomically Compare and exchange an unsigned 64-bit value, additionally
* passing back old value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pu64 Pointer to the 64-bit variable to update.
* @param u64New The 64-bit value to assign to *pu64.
* @param u64Old The value to compare with.
* @param pu64Old Pointer store the old value at.
*/
#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old);
#else
DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old)
{
# elif defined(RT_ARCH_AMD64)
"setz %1\n\t"
: "=m" (*pu64),
"=qm" (u8Ret),
"=a" (*pu64Old)
: "r" (u64New),
"a" (u64Old),
"m" (*pu64));
return (bool)u8Ret;
# else
bool fRet;
{
}
return fRet;
# endif
# else /* !RT_ARCH_AMD64 */
/* NB: this code uses a memory clobber description, because the clean
* solution with an output value for *pu64 makes gcc run out of registers.
* This will cause suboptimal code, and anyone with a better solution is
* welcome to improve this. */
"lock; cmpxchg8b %3\n\t"
"xchgl %%ebx, %1\n\t"
: "=A" (u64Ret)
"m" (*pu64),
"0" (u64Old)
: "memory" );
# else /* !PIC */
: "=A" (u64Ret),
"=m" (*pu64)
"m" (*pu64),
"0" (u64Old));
# endif
# else
{
}
return !!u32Ret;
# endif
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Atomically Compare and exchange a signed 64-bit value, additionally
* passing back old value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param pi64 Pointer to the 64-bit variable to update.
* @param i64 The 64-bit value to assign to *pu64.
* @param i64Old The value to compare with.
* @param pi64Old Pointer store the old value at.
*/
DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old, int64_t *pi64Old)
{
return ASMAtomicCmpXchgExU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t *)pi64Old);
}
/** @def ASMAtomicCmpXchgExHandle
* Atomically Compare and Exchange a typical IPRT handle value, ordered.
*
* @param ph Pointer to the value to update.
* @param hNew The new value to assigned to *pu.
* @param hOld The old value to *pu compare with.
* @param fRc Where to store the result.
* @param phOldVal Pointer to where to store the old value.
*
* @remarks This doesn't currently work for all handles (like RTFILE).
*/
#if HC_ARCH_BITS == 32
do { \
(fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(puOldVal)); \
} while (0)
do { \
(fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(puOldVal)); \
} while (0)
#else
#endif
/** @def ASMAtomicCmpXchgExSize
* Atomically Compare and Exchange a value which size might differ
* between platforms or compilers. Additionally passes back old value.
*
* @param pu Pointer to the value to update.
* @param uNew The new value to assigned to *pu.
* @param uOld The old value to *pu compare with.
* @param fRc Where to store the result.
* @param puOldVal Pointer to where to store the old value.
*/
do { \
switch (sizeof(*(pu))) { \
case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
break; \
case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
break; \
(fRc) = false; \
(uOldVal) = 0; \
break; \
} \
} while (0)
/**
* Atomically Compare and Exchange a pointer value, additionally
* passing back old value, ordered.
*
* @returns true if xchg was done.
* @returns false if xchg wasn't done.
*
* @param ppv Pointer to the value to update.
* @param pvNew The new value to assigned to *ppv.
* @param pvOld The old value to *ppv compare with.
* @param ppvOld Pointer store the old value at.
*/
DECLINLINE(bool) ASMAtomicCmpXchgExPtr(void * volatile *ppv, const void *pvNew, const void *pvOld, void **ppvOld)
{
#if ARCH_BITS == 32
return ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
return ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
#else
# error "ARCH_BITS is bogus"
#endif
}
/**
* Atomically exchanges and adds to a 32-bit value, ordered.
*
* @returns The old value.
* @param pu32 Pointer to the value.
* @param u32 Number to add.
*/
#else
{
return u32;
: "=r" (u32),
"=m" (*pu32)
: "0" (u32),
"m" (*pu32)
: "memory");
return u32;
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
return u32;
# endif
}
#endif
/**
* Atomically exchanges and adds to a signed 32-bit value, ordered.
*
* @returns The old value.
* @param pi32 Pointer to the value.
* @param i32 Number to add.
*/
{
}
/**
* Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
*
* @returns The old value.
* @param pu32 Pointer to the value.
* @param u32 Number to subtract.
*/
{
}
/**
* Atomically exchanges and subtracts to a signed 32-bit value, ordered.
*
* @returns The old value.
* @param pi32 Pointer to the value.
* @param i32 Number to subtract.
*/
{
}
/**
* Atomically increment a 32-bit value, ordered.
*
* @returns The new value.
* @param pu32 Pointer to the value to increment.
*/
#else
{
return u32;
: "=r" (u32),
"=m" (*pu32)
: "0" (1),
"m" (*pu32)
: "memory");
return u32+1;
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
return u32+1;
# endif
}
#endif
/**
* Atomically increment a signed 32-bit value, ordered.
*
* @returns The new value.
* @param pi32 Pointer to the value to increment.
*/
{
}
/**
* Atomically decrement an unsigned 32-bit value, ordered.
*
* @returns The new value.
* @param pu32 Pointer to the value to decrement.
*/
#else
{
return u32;
: "=r" (u32),
"=m" (*pu32)
: "0" (-1),
"m" (*pu32)
: "memory");
return u32-1;
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
return u32-1;
# endif
}
#endif
/**
* Atomically decrement a signed 32-bit value, ordered.
*
* @returns The new value.
* @param pi32 Pointer to the value to decrement.
*/
{
}
/**
* Atomically Or an unsigned 32-bit value, ordered.
*
* @param pu32 Pointer to the pointer variable to OR u32 with.
* @param u32 The value to OR *pu32 with.
*/
#else
{
: "=m" (*pu32)
: "ir" (u32),
"m" (*pu32));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Atomically Or a signed 32-bit value, ordered.
*
* @param pi32 Pointer to the pointer variable to OR u32 with.
* @param i32 The value to OR *pu32 with.
*/
{
}
/**
* Atomically And an unsigned 32-bit value, ordered.
*
* @param pu32 Pointer to the pointer variable to AND u32 with.
* @param u32 The value to AND *pu32 with.
*/
#else
{
: "=m" (*pu32)
: "ir" (u32),
"m" (*pu32));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Atomically And a signed 32-bit value, ordered.
*
* @param pi32 Pointer to the pointer variable to AND i32 with.
* @param i32 The value to AND *pi32 with.
*/
{
}
/**
* Serialize Instruction.
*/
DECLASM(void) ASMSerializeInstruction(void);
#else
DECLINLINE(void) ASMSerializeInstruction(void)
{
RTCCUINTREG xAX = 0;
# ifdef RT_ARCH_AMD64
__asm__ ("cpuid"
: "=a" (xAX)
: "0" (xAX)
: "rbx", "rcx", "rdx");
__asm__ ("push %%ebx\n\t"
"cpuid\n\t"
"pop %%ebx\n\t"
: "=a" (xAX)
: "0" (xAX)
: "ecx", "edx");
# else
__asm__ ("cpuid"
: "=a" (xAX)
: "0" (xAX)
: "ebx", "ecx", "edx");
# endif
int aInfo[4];
# else
{
}
# endif
}
#endif
/**
* Requires the X86_CPUID_FEATURE_EDX_SSE2 CPUID bit set.
*/
DECLINLINE(void) ASMMemoryFenceSSE2(void)
{
_mm_mfence();
#else
{
_emit 0x0f
_emit 0xae
_emit 0xf0
}
#endif
}
/**
* Memory store fence, waits for any writes to complete.
* Requires the X86_CPUID_FEATURE_EDX_SSE CPUID bit set.
*/
DECLINLINE(void) ASMWriteFenceSSE(void)
{
_mm_sfence();
#else
{
_emit 0x0f
_emit 0xae
_emit 0xf8
}
#endif
}
/**
* Memory load fence, waits for any pending reads to complete.
* Requires the X86_CPUID_FEATURE_EDX_SSE2 CPUID bit set.
*/
DECLINLINE(void) ASMReadFenceSSE2(void)
{
_mm_lfence();
#else
{
_emit 0x0f
_emit 0xae
_emit 0xe8
}
#endif
}
/**
* Memory fence, waits for any pending writes and reads to complete.
*/
DECLINLINE(void) ASMMemoryFence(void)
{
/** @todo use mfence? check if all cpus we care for support it. */
ASMAtomicXchgU32(&u32, 0);
}
/**
* Write fence, waits for any pending writes to complete.
*/
DECLINLINE(void) ASMWriteFence(void)
{
/** @todo use sfence? check if all cpus we care for support it. */
}
/**
* Read fence, waits for any pending reads to complete.
*/
DECLINLINE(void) ASMReadFence(void)
{
/** @todo use lfence? check if all cpus we care for support it. */
}
/**
* Atomically reads an unsigned 8-bit value, ordered.
*
* @returns Current *pu8 value
* @param pu8 Pointer to the 8-bit variable to read.
*/
{
return *pu8; /* byte reads are atomic on x86 */
}
/**
* Atomically reads an unsigned 8-bit value, unordered.
*
* @returns Current *pu8 value
* @param pu8 Pointer to the 8-bit variable to read.
*/
{
return *pu8; /* byte reads are atomic on x86 */
}
/**
* Atomically reads a signed 8-bit value, ordered.
*
* @returns Current *pi8 value
* @param pi8 Pointer to the 8-bit variable to read.
*/
{
return *pi8; /* byte reads are atomic on x86 */
}
/**
* Atomically reads a signed 8-bit value, unordered.
*
* @returns Current *pi8 value
* @param pi8 Pointer to the 8-bit variable to read.
*/
{
return *pi8; /* byte reads are atomic on x86 */
}
/**
* Atomically reads an unsigned 16-bit value, ordered.
*
* @returns Current *pu16 value
* @param pu16 Pointer to the 16-bit variable to read.
*/
{
return *pu16;
}
/**
* Atomically reads an unsigned 16-bit value, unordered.
*
* @returns Current *pu16 value
* @param pu16 Pointer to the 16-bit variable to read.
*/
{
return *pu16;
}
/**
* Atomically reads a signed 16-bit value, ordered.
*
* @returns Current *pi16 value
* @param pi16 Pointer to the 16-bit variable to read.
*/
{
return *pi16;
}
/**
* Atomically reads a signed 16-bit value, unordered.
*
* @returns Current *pi16 value
* @param pi16 Pointer to the 16-bit variable to read.
*/
{
return *pi16;
}
/**
* Atomically reads an unsigned 32-bit value, ordered.
*
* @returns Current *pu32 value
* @param pu32 Pointer to the 32-bit variable to read.
*/
{
return *pu32;
}
/**
* Atomically reads an unsigned 32-bit value, unordered.
*
* @returns Current *pu32 value
* @param pu32 Pointer to the 32-bit variable to read.
*/
{
return *pu32;
}
/**
* Atomically reads a signed 32-bit value, ordered.
*
* @returns Current *pi32 value
* @param pi32 Pointer to the 32-bit variable to read.
*/
{
return *pi32;
}
/**
* Atomically reads a signed 32-bit value, unordered.
*
* @returns Current *pi32 value
* @param pi32 Pointer to the 32-bit variable to read.
*/
{
return *pi32;
}
/**
* Atomically reads an unsigned 64-bit value, ordered.
*
* @returns Current *pu64 value
* @param pu64 Pointer to the 64-bit variable to read.
* The memory pointed to must be writable.
* @remark This will fault if the memory is read-only!
*/
#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
#else
{
# ifdef RT_ARCH_AMD64
/*# if RT_INLINE_ASM_GNU_STYLE
__asm__ __volatile__( "mfence\n\t"
"movq %1, %0\n\t"
: "=r" (u64)
: "m" (*pu64));
# else
__asm
{
mfence
mov rdx, [pu64]
mov rax, [rdx]
mov [u64], rax
}
# endif*/
# else /* !RT_ARCH_AMD64 */
"lock; cmpxchg8b (%5)\n\t"
"movl %3, %%ebx\n\t"
: "=A" (u64),
"+m" (*pu64)
# else
"=m" (*pu64)
# endif
: "0" (0),
"m" (u32EBX),
"c" (0),
"S" (pu64));
# else /* !PIC */
: "=A" (u64),
"+m" (*pu64)
: "0" (0),
"b" (0),
"c" (0));
# endif
# else
{
}
# endif
# endif /* !RT_ARCH_AMD64 */
return u64;
}
#endif
/**
* Atomically reads an unsigned 64-bit value, unordered.
*
* @returns Current *pu64 value
* @param pu64 Pointer to the 64-bit variable to read.
* The memory pointed to must be writable.
* @remark This will fault if the memory is read-only!
*/
#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
#else
{
# ifdef RT_ARCH_AMD64
/*# if RT_INLINE_ASM_GNU_STYLE
Assert(!((uintptr_t)pu64 & 7));
__asm__ __volatile__("movq %1, %0\n\t"
: "=r" (u64)
: "m" (*pu64));
# else
__asm
{
mov rdx, [pu64]
mov rax, [rdx]
mov [u64], rax
}
# endif */
# else /* !RT_ARCH_AMD64 */
"xor %%ecx,%%ecx\n\t"
"xor %%edx,%%edx\n\t"
"xchgl %%ebx, %3\n\t"
"lock; cmpxchg8b (%4)\n\t"
"movl %3, %%ebx\n\t"
: "=A" (u64),
"+m" (*pu64),
# else
"=m" (*pu64),
# endif
"=c" (u32Spill)
: "m" (u32EBX),
"S" (pu64));
# else /* !PIC */
: "=A" (u64),
"+m" (*pu64)
: "0" (0),
"b" (0),
"c" (0));
# endif
# else
{
}
# endif
# endif /* !RT_ARCH_AMD64 */
return u64;
}
#endif
/**
* Atomically reads a signed 64-bit value, ordered.
*
* @returns Current *pi64 value
* @param pi64 Pointer to the 64-bit variable to read.
* The memory pointed to must be writable.
* @remark This will fault if the memory is read-only!
*/
{
}
/**
* Atomically reads a signed 64-bit value, unordered.
*
* @returns Current *pi64 value
* @param pi64 Pointer to the 64-bit variable to read.
* The memory pointed to must be writable.
* @remark This will fault if the memory is read-only!
*/
{
}
/**
* Atomically reads a pointer value, ordered.
*
* @returns Current *pv value
* @param ppv Pointer to the pointer variable to read.
*/
{
#if ARCH_BITS == 32
#else
# error "ARCH_BITS is bogus"
#endif
}
/**
* Atomically reads a pointer value, unordered.
*
* @returns Current *pv value
* @param ppv Pointer to the pointer variable to read.
*/
{
#if ARCH_BITS == 32
#else
# error "ARCH_BITS is bogus"
#endif
}
/**
* Atomically reads a boolean value, ordered.
*
* @returns Current *pf value
* @param pf Pointer to the boolean variable to read.
*/
{
return *pf; /* byte reads are atomic on x86 */
}
/**
* Atomically reads a boolean value, unordered.
*
* @returns Current *pf value
* @param pf Pointer to the boolean variable to read.
*/
{
return *pf; /* byte reads are atomic on x86 */
}
/**
* Atomically read a typical IPRT handle value, ordered.
*
* @param ph Pointer to the handle variable to read.
* @param phRes Where to store the result.
*
* @remarks This doesn't currently work for all handles (like RTFILE).
*/
#if HC_ARCH_BITS == 32
do { \
} while (0)
do { \
} while (0)
#else
#endif
/**
* Atomically read a typical IPRT handle value, unordered.
*
* @param ph Pointer to the handle variable to read.
* @param phRes Where to store the result.
*
* @remarks This doesn't currently work for all handles (like RTFILE).
*/
#if HC_ARCH_BITS == 32
do { \
} while (0)
do { \
} while (0)
#else
#endif
/**
* Atomically read a value which size might differ
* between platforms or compilers, ordered.
*
* @param pu Pointer to the variable to update.
* @param puRes Where to store the result.
*/
do { \
switch (sizeof(*(pu))) { \
} \
} while (0)
/**
* Atomically read a value which size might differ
* between platforms or compilers, unordered.
*
* @param pu Pointer to the variable to read.
* @param puRes Where to store the result.
*/
do { \
switch (sizeof(*(pu))) { \
} \
} while (0)
/**
* Atomically writes an unsigned 8-bit value, ordered.
*
* @param pu8 Pointer to the 8-bit variable.
* @param u8 The 8-bit value to assign to *pu8.
*/
{
}
/**
* Atomically writes an unsigned 8-bit value, unordered.
*
* @param pu8 Pointer to the 8-bit variable.
* @param u8 The 8-bit value to assign to *pu8.
*/
{
}
/**
* Atomically writes a signed 8-bit value, ordered.
*
* @param pi8 Pointer to the 8-bit variable to read.
* @param i8 The 8-bit value to assign to *pi8.
*/
{
}
/**
* Atomically writes a signed 8-bit value, unordered.
*
* @param pi8 Pointer to the 8-bit variable to read.
* @param i8 The 8-bit value to assign to *pi8.
*/
{
}
/**
* Atomically writes an unsigned 16-bit value, ordered.
*
* @param pu16 Pointer to the 16-bit variable.
* @param u16 The 16-bit value to assign to *pu16.
*/
{
}
/**
* Atomically writes an unsigned 16-bit value, unordered.
*
* @param pu16 Pointer to the 16-bit variable.
* @param u16 The 16-bit value to assign to *pu16.
*/
{
}
/**
* Atomically writes a signed 16-bit value, ordered.
*
* @param pi16 Pointer to the 16-bit variable to read.
* @param i16 The 16-bit value to assign to *pi16.
*/
{
}
/**
* Atomically writes a signed 16-bit value, unordered.
*
* @param pi16 Pointer to the 16-bit variable to read.
* @param i16 The 16-bit value to assign to *pi16.
*/
{
}
/**
* Atomically writes an unsigned 32-bit value, ordered.
*
* @param pu32 Pointer to the 32-bit variable.
* @param u32 The 32-bit value to assign to *pu32.
*/
{
}
/**
* Atomically writes an unsigned 32-bit value, unordered.
*
* @param pu32 Pointer to the 32-bit variable.
* @param u32 The 32-bit value to assign to *pu32.
*/
{
}
/**
* Atomically writes a signed 32-bit value, ordered.
*
* @param pi32 Pointer to the 32-bit variable to read.
* @param i32 The 32-bit value to assign to *pi32.
*/
{
}
/**
* Atomically writes a signed 32-bit value, unordered.
*
* @param pi32 Pointer to the 32-bit variable to read.
* @param i32 The 32-bit value to assign to *pi32.
*/
{
}
/**
* Atomically writes an unsigned 64-bit value, ordered.
*
* @param pu64 Pointer to the 64-bit variable.
* @param u64 The 64-bit value to assign to *pu64.
*/
{
}
/**
* Atomically writes an unsigned 64-bit value, unordered.
*
* @param pu64 Pointer to the 64-bit variable.
* @param u64 The 64-bit value to assign to *pu64.
*/
{
#if ARCH_BITS == 64
#else
#endif
}
/**
* Atomically writes a signed 64-bit value, ordered.
*
* @param pi64 Pointer to the 64-bit variable.
* @param i64 The 64-bit value to assign to *pi64.
*/
{
}
/**
* Atomically writes a signed 64-bit value, unordered.
*
* @param pi64 Pointer to the 64-bit variable.
* @param i64 The 64-bit value to assign to *pi64.
*/
{
#if ARCH_BITS == 64
#else
#endif
}
/**
* Atomically writes a boolean value, unordered.
*
* @param pf Pointer to the boolean variable.
* @param f The boolean value to assign to *pf.
*/
{
}
/**
* Atomically writes a boolean value, unordered.
*
* @param pf Pointer to the boolean variable.
* @param f The boolean value to assign to *pf.
*/
{
*pf = f; /* byte writes are atomic on x86 */
}
/**
* Atomically writes a pointer value, ordered.
*
* @returns Current *pv value
* @param ppv Pointer to the pointer variable.
* @param pv The pointer value to assigne to *ppv.
*/
{
#if ARCH_BITS == 32
#else
# error "ARCH_BITS is bogus"
#endif
}
/**
* Atomically writes a pointer value, unordered.
*
* @returns Current *pv value
* @param ppv Pointer to the pointer variable.
* @param pv The pointer value to assigne to *ppv.
*/
{
#if ARCH_BITS == 32
#else
# error "ARCH_BITS is bogus"
#endif
}
/**
* Atomically write a typical IPRT handle value, ordered.
*
* @param ph Pointer to the variable to update.
* @param hNew The value to assign to *ph.
*
* @remarks This doesn't currently work for all handles (like RTFILE).
*/
#if HC_ARCH_BITS == 32
do { \
} while (0)
do { \
} while (0)
#else
#endif
/**
* Atomically write a typical IPRT handle value, unordered.
*
* @param ph Pointer to the variable to update.
* @param hNew The value to assign to *ph.
*
* @remarks This doesn't currently work for all handles (like RTFILE).
*/
#if HC_ARCH_BITS == 32
do { \
} while (0)
do { \
} while (0)
#else
#endif
/**
* Atomically write a value which size might differ
* between platforms or compilers, ordered.
*
* @param pu Pointer to the variable to update.
* @param uNew The value to assign to *pu.
*/
do { \
switch (sizeof(*(pu))) { \
} \
} while (0)
/**
* Atomically write a value which size might differ
* between platforms or compilers, unordered.
*
* @param pu Pointer to the variable to update.
* @param uNew The value to assign to *pu.
*/
do { \
switch (sizeof(*(pu))) { \
} \
} while (0)
/**
* Invalidate page.
*
* @param pv Address of the page to invalidate.
*/
#else
{
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Write back the internal caches and invalidate them.
*/
DECLASM(void) ASMWriteBackAndInvalidateCaches(void);
#else
DECLINLINE(void) ASMWriteBackAndInvalidateCaches(void)
{
__wbinvd();
# else
{
}
# endif
}
#endif
/**
* Invalidate internal and (perhaps) external caches without first
* flushing dirty cache lines. Use with extreme care.
*/
DECLASM(void) ASMInvalidateInternalCaches(void);
#else
DECLINLINE(void) ASMInvalidateInternalCaches(void)
{
# else
{
}
# endif
}
#endif
#if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
# if PAGE_SIZE != 0x1000
# error "PAGE_SIZE is not 0x1000!"
# endif
#endif
/**
* Zeros a 4K memory page.
*
* @param pv Pointer to the memory block. This must be page aligned.
*/
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# ifdef RT_ARCH_AMD64
: "=D" (pv),
"=c" (uDummy)
: "0" (pv),
"c" (0x1000 >> 3),
"a" (0)
: "memory");
# else
: "=D" (pv),
"=c" (uDummy)
: "0" (pv),
"c" (0x1000 >> 2),
"a" (0)
: "memory");
# endif
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
# endif
/**
* Zeros a memory block with a 32-bit aligned size.
*
* @param pv Pointer to the memory block.
* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
*/
#else
{
# ifdef RT_ARCH_AMD64
if (!(cb & 7))
else
# endif
: "=D" (pv),
"=c" (cb)
: "0" (pv),
"a" (0)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Fills a memory block with a 32-bit aligned size.
*
* @param pv Pointer to the memory block.
* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
* @param u32 The value to fill with.
*/
#else
{
# ifdef RT_ARCH_AMD64
if (!(cb & 7))
else
# endif
: "=D" (pv),
"=c" (cb)
: "0" (pv),
"a" (u32)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Checks if a memory page is all zeros.
*
* @returns true / false.
*
* @param pvPage Pointer to the page. Must be aligned on 16 byte
* boundrary
*/
{
# if 0 /*RT_INLINE_ASM_GNU_STYLE - this is actually slower... */
union { RTCCUINTREG r; bool f; } uAX;
# ifdef RT_ARCH_AMD64
"scasq\n\t"
# else
"scasl\n\t"
# endif
"setnc %%al\n\t"
: "=&c" (xCX),
"=&D" (xDI),
"=&a" (uAX.r)
: "mr" (pvPage),
# ifdef RT_ARCH_AMD64
"0" (0x1000/8),
# else
"0" (0x1000/4),
# endif
"1" (pvPage),
"2" (0));
return uAX.f;
# else
for (;;)
{
if (puPtr[0]) return false;
if (puPtr[4]) return false;
if (puPtr[2]) return false;
if (puPtr[6]) return false;
if (puPtr[1]) return false;
if (puPtr[5]) return false;
if (puPtr[3]) return false;
if (puPtr[7]) return false;
if (!--cLeft)
return true;
puPtr += 8;
}
return true;
# endif
}
/**
* Checks if a memory block is filled with the specified byte.
*
* This is a sort of inverted memchr.
*
* @returns Pointer to the byte which doesn't equal u8.
* @returns NULL if all equal to u8.
*
* @param pv Pointer to the memory block.
* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
* @param u8 The value it's supposed to be filled with.
*
* @todo Fix name, it is a predicate function but it's not returning boolean!
*/
#else
{
/** @todo rewrite this in inline assembly? */
return (void *)pb;
return NULL;
}
#endif
/**
* Checks if a memory block is filled with the specified 32-bit value.
*
* This is a sort of inverted memchr.
*
* @returns Pointer to the first value which doesn't equal u32.
* @returns NULL if all equal to u32.
*
* @param pv Pointer to the memory block.
* @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
* @param u32 The value it's supposed to be filled with.
*
* @todo Fix name, it is a predicate function but it's not returning boolean!
*/
#else
{
/** @todo rewrite this in inline assembly? */
return NULL;
}
#endif
/**
* Multiplies two unsigned 32-bit values returning an unsigned 64-bit result.
*
* @returns u32F1 * u32F2.
*/
#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
#else
{
# ifdef RT_ARCH_AMD64
# else /* !RT_ARCH_AMD64 */
: "=A" (u64)
# else
{
}
# endif
return u64;
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Multiplies two signed 32-bit values returning a signed 64-bit result.
*
* @returns u32F1 * u32F2.
*/
#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
#else
{
# ifdef RT_ARCH_AMD64
# else /* !RT_ARCH_AMD64 */
: "=A" (i64)
# else
{
}
# endif
return i64;
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Divides a 64-bit unsigned by a 32-bit unsigned returning an unsigned 32-bit result.
*
* @returns u64 / u32.
*/
#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
#else
{
# ifdef RT_ARCH_AMD64
# else /* !RT_ARCH_AMD64 */
# else
{
}
# endif
return u32;
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Divides a 64-bit signed by a 32-bit signed returning a signed 32-bit result.
*
* @returns u64 / u32.
*/
#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
#else
{
# ifdef RT_ARCH_AMD64
# else /* !RT_ARCH_AMD64 */
# else
{
}
# endif
return i32;
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Performs 64-bit unsigned by a 32-bit unsigned division with a 32-bit unsigned result,
* returning the rest.
*
* @returns u64 % u32.
*
* @remarks It is important that the result is <= UINT32_MAX or we'll overflow and crash.
*/
#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
#else
{
# ifdef RT_ARCH_AMD64
# else /* !RT_ARCH_AMD64 */
# else
{
}
# endif
return u32;
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Performs 64-bit signed by a 32-bit signed division with a 32-bit signed result,
* returning the rest.
*
* @returns u64 % u32.
*
* @remarks It is important that the result is <= UINT32_MAX or we'll overflow and crash.
*/
#if RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)
#else
{
# ifdef RT_ARCH_AMD64
# else /* !RT_ARCH_AMD64 */
# else
{
}
# endif
return i32;
# endif /* !RT_ARCH_AMD64 */
}
#endif
/**
* Multiple a 64-bit by a 32-bit integer and divide the result by a 32-bit integer
* using a 96 bit intermediate result.
* @note Don't use 64-bit C arithmetic here since some gcc compilers generate references to
* __udivdi3 and __umoddi3 even if this inline function is not used.
*
* @returns (u64A * u32B) / u32C.
* @param u64A The 64-bit value.
* @param u32B The 32-bit value to multiple by A.
* @param u32C The 32-bit value to divide A*B by.
*/
#if RT_INLINE_ASM_EXTERNAL || !defined(__GNUC__)
#else
{
# ifdef RT_ARCH_AMD64
"divq %3\n\t"
: "=a" (u64Result),
"=d" (u64Spill)
"0" (u64A),
"1" (0));
return u64Result;
# else
edx = u64Lo.hi = (u64A.lo * u32B).hi */
"xchg %%eax,%%esi \n\t" /* esi = u64Lo.lo
eax = u64A.hi */
"xchg %%edx,%%edi \n\t" /* edi = u64Low.hi
edx = u32C */
"xchg %%edx,%%ecx \n\t" /* ecx = u32C
edx = u32B */
"mull %%edx \n\t" /* eax = u64Hi.lo = (u64A.hi * u32B).lo
"addl %%edi,%%eax \n\t" /* u64Hi.lo += u64Lo.hi */
"adcl $0,%%edx \n\t" /* u64Hi.hi += carry */
"divl %%ecx \n\t" /* eax = u64Hi / u32C
edx = u64Hi % u32C */
"movl %%eax,%%edi \n\t" /* edi = u64Result.hi = u64Hi / u32C */
"movl %%esi,%%eax \n\t" /* eax = u64Lo.lo */
"divl %%ecx \n\t" /* u64Result.lo */
"movl %%edi,%%edx \n\t" /* u64Result.hi */
"c"(u32B),
"D"(u32C));
return u64Result;
# endif
# else
RTUINT64U u;
return u.u;
# endif
}
#endif
/**
* Probes a byte pointer for read access.
*
* While the function will not fault if the byte is not read accessible,
* the idea is to do this in a safe place like before acquiring locks
* and such like.
*
* Also, this functions guarantees that an eager compiler is not going
* to optimize the probing away.
*
* @param pvByte Pointer to the byte.
*/
#else
{
/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
: "=r" (u8)
: "r" (pvByte));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return u8;
}
#endif
/**
* Probes a buffer for read access page by page.
*
* While the function will fault if the buffer is not fully read
* accessible, the idea is to do this in a safe place like before
* acquiring locks and such like.
*
* Also, this functions guarantees that an eager compiler is not going
* to optimize the probing away.
*
* @param pvBuf Pointer to the buffer.
* @param cbBuf The size of the buffer in bytes. Must be >= 1.
*/
{
/** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
/* the first byte */
/* the pages in between pages. */
{
}
/* the last byte */
}
/** @def ASMBreakpoint
* Debugger Breakpoint.
* @remark In the gnu world we add a nop instruction after the int3 to
* force gdb to remain at the int3 source line.
* @remark The L4 kernel will try make sense of the breakpoint, thus the jmp.
* @internal
*/
# ifndef __L4ENV__
# else
# endif
#else
# define ASMBreakpoint() __debugbreak()
#endif
/** @defgroup grp_inline_bits Bit Operations
* @{
*/
/**
* Sets a bit in a bitmap.
*
* @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
* @param iBit The bit to set.
*
* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
* However, doing so will yield better performance as well as avoiding
* traps accessing the last bits in the bitmap.
*/
#else
{
: "=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Atomically sets a bit in a bitmap, ordered.
*
* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
* the memory access isn't atomic!
* @param iBit The bit to set.
*/
#else
{
: "=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Clears a bit in a bitmap.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBit The bit to clear.
*
* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
* However, doing so will yield better performance as well as avoiding
* traps accessing the last bits in the bitmap.
*/
#else
{
: "=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Atomically clears a bit in a bitmap, ordered.
*
* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
* the memory access isn't atomic!
* @param iBit The bit to toggle set.
* @remarks No memory barrier, take care on smp.
*/
#else
{
: "=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Toggles a bit in a bitmap.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBit The bit to toggle.
*
* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
* However, doing so will yield better performance as well as avoiding
* traps accessing the last bits in the bitmap.
*/
#else
{
: "=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Atomically toggles a bit in a bitmap, ordered.
*
* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
* the memory access isn't atomic!
* @param iBit The bit to test and set.
*/
#else
{
: "=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
}
#endif
/**
* Tests and sets a bit in a bitmap.
*
* @returns true if the bit was set.
* @returns false if the bit was clear.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBit The bit to test and set.
*
* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
* However, doing so will yield better performance as well as avoiding
* traps accessing the last bits in the bitmap.
*/
#else
{
"setc %b0\n\t"
"andl $1, %0\n\t"
"=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return rc.f;
}
#endif
/**
* Atomically tests and sets a bit in a bitmap, ordered.
*
* @returns true if the bit was set.
* @returns false if the bit was clear.
*
* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
* the memory access isn't atomic!
* @param iBit The bit to set.
*/
#else
{
"setc %b0\n\t"
"andl $1, %0\n\t"
"=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return rc.f;
}
#endif
/**
* Tests and clears a bit in a bitmap.
*
* @returns true if the bit was set.
* @returns false if the bit was clear.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBit The bit to test and clear.
*
* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
* However, doing so will yield better performance as well as avoiding
* traps accessing the last bits in the bitmap.
*/
#else
{
"setc %b0\n\t"
"andl $1, %0\n\t"
"=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return rc.f;
}
#endif
/**
* Atomically tests and clears a bit in a bitmap, ordered.
*
* @returns true if the bit was set.
* @returns false if the bit was clear.
*
* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
* the memory access isn't atomic!
* @param iBit The bit to test and clear.
*
* @remarks No memory barrier, take care on smp.
*/
#else
{
"setc %b0\n\t"
"andl $1, %0\n\t"
"=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return rc.f;
}
#endif
/**
* Tests and toggles a bit in a bitmap.
*
* @returns true if the bit was set.
* @returns false if the bit was clear.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBit The bit to test and toggle.
*
* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
* However, doing so will yield better performance as well as avoiding
* traps accessing the last bits in the bitmap.
*/
#else
{
"setc %b0\n\t"
"andl $1, %0\n\t"
"=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return rc.f;
}
#endif
/**
* Atomically tests and toggles a bit in a bitmap, ordered.
*
* @returns true if the bit was set.
* @returns false if the bit was clear.
*
* @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
* the memory access isn't atomic!
* @param iBit The bit to test and toggle.
*/
#else
{
"setc %b0\n\t"
"andl $1, %0\n\t"
"=m" (*(volatile long *)pvBitmap)
: "Ir" (iBit),
"m" (*(volatile long *)pvBitmap)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return rc.f;
}
#endif
/**
* Tests if a bit in a bitmap is set.
*
* @returns true if the bit is set.
* @returns false if the bit is clear.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBit The bit to test.
*
* @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
* However, doing so will yield better performance as well as avoiding
* traps accessing the last bits in the bitmap.
*/
#else
{
"setc %b0\n\t"
"andl $1, %0\n\t"
: "m" (*(const volatile long *)pvBitmap),
"Ir" (iBit)
: "memory");
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
}
# endif
return rc.f;
}
#endif
/**
* Clears a bit range within a bitmap.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBitStart The First bit to clear.
* @param iBitEnd The first bit not to clear.
*/
{
{
else
{
/* bits in first dword. */
if (iBitStart & 31)
{
pu32++;
}
/* whole dword. */
/* bits in last dword. */
if (iBitEnd & 31)
{
}
}
}
}
/**
* Sets a bit range within a bitmap.
*
* @param pvBitmap Pointer to the bitmap.
* @param iBitStart The First bit to set.
* @param iBitEnd The first bit not to set.
*/
{
{
else
{
/* bits in first dword. */
if (iBitStart & 31)
{
pu32++;
}
/* whole dword. */
/* bits in last dword. */
if (iBitEnd & 31)
{
}
}
}
}
/**
* Finds the first clear bit in a bitmap.
*
* @returns Index of the first zero bit.
* @returns -1 if no clear bit was found.
* @param pvBitmap Pointer to the bitmap.
* @param cBits The number of bits in the bitmap. Multiple of 32.
*/
#else
{
if (cBits)
{
"je 1f\n\t"
# ifdef RT_ARCH_AMD64
"lea -4(%%rdi), %%rdi\n\t"
"xorl (%%rdi), %%eax\n\t"
"subq %5, %%rdi\n\t"
# else
"lea -4(%%edi), %%edi\n\t"
"xorl (%%edi), %%eax\n\t"
"subl %5, %%edi\n\t"
# endif
"shll $3, %%edi\n\t"
"bsfl %%eax, %%edx\n\t"
"addl %%edi, %%edx\n\t"
"1:\t\n"
: "=d" (iBit),
"=&c" (uECX),
"=&D" (uEDI),
"=&a" (uEAX)
: "0" (0xffffffff),
"mr" (pvBitmap),
"2" (pvBitmap),
"3" (0xffffffff));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# ifdef RT_ARCH_AMD64
# else
# endif
done:
}
# endif
return iBit;
}
return -1;
}
#endif
/**
* Finds the next clear bit in a bitmap.
*
* @returns Index of the first zero bit.
* @returns -1 if no clear bit was found.
* @param pvBitmap Pointer to the bitmap.
* @param cBits The number of bits in the bitmap. Multiple of 32.
* @param iBitPrev The bit returned from the last search.
* The search will start at iBitPrev + 1.
*/
#else
{
if (iBit)
{
/*
* Inspect the 32-bit word containing the unaligned bit.
*/
unsigned long ulBit = 0;
# else
"jnz 1f\n\t"
"movl $-1, %0\n\t"
"1:\n\t"
: "=r" (iBit)
: "r" (u32));
# else
{
done:
}
# endif
if (iBit >= 0)
# endif
/*
* Skip ahead and see if there is anything left to search.
*/
iBitPrev |= 31;
iBitPrev++;
return -1;
}
/*
* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
*/
if (iBit >= 0)
return iBit;
}
#endif
/**
* Finds the first set bit in a bitmap.
*
* @returns Index of the first set bit.
* @returns -1 if no clear bit was found.
* @param pvBitmap Pointer to the bitmap.
* @param cBits The number of bits in the bitmap. Multiple of 32.
*/
#else
{
if (cBits)
{
"je 1f\n\t"
# ifdef RT_ARCH_AMD64
"lea -4(%%rdi), %%rdi\n\t"
"movl (%%rdi), %%eax\n\t"
"subq %5, %%rdi\n\t"
# else
"lea -4(%%edi), %%edi\n\t"
"movl (%%edi), %%eax\n\t"
"subl %5, %%edi\n\t"
# endif
"shll $3, %%edi\n\t"
"bsfl %%eax, %%edx\n\t"
"addl %%edi, %%edx\n\t"
"1:\t\n"
: "=d" (iBit),
"=&c" (uECX),
"=&D" (uEDI),
"=&a" (uEAX)
: "0" (0xffffffff),
"mr" (pvBitmap),
"2" (pvBitmap),
"3" (0));
# else
{
# ifdef RT_ARCH_AMD64
# else
# endif
# ifdef RT_ARCH_AMD64
# else
# endif
done:
}
# endif
return iBit;
}
return -1;
}
#endif
/**
* Finds the next set bit in a bitmap.
*
* @returns Index of the next set bit.
* @returns -1 if no set bit was found.
* @param pvBitmap Pointer to the bitmap.
* @param cBits The number of bits in the bitmap. Multiple of 32.
* @param iBitPrev The bit returned from the last search.
* The search will start at iBitPrev + 1.
*/
#else
{
if (iBit)
{
/*
* Inspect the 32-bit word containing the unaligned bit.
*/
unsigned long ulBit = 0;
# else
"jnz 1f\n\t"
"movl $-1, %0\n\t"
"1:\n\t"
: "=r" (iBit)
: "r" (u32));
# else
{
done:
}
# endif
if (iBit >= 0)
# endif
/*
* Skip ahead and see if there is anything left to search.
*/
iBitPrev |= 31;
iBitPrev++;
return -1;
}
/*
* 32-bit aligned search, let ASMBitFirstClear do the dirty work.
*/
if (iBit >= 0)
return iBit;
}
#endif
/**
* Finds the first bit which is set in the given 32-bit integer.
* Bits are numbered from 1 (least significant) to 32.
*
* @returns index [1..32] of the first set bit.
* @returns 0 if all bits are cleared.
* @param u32 Integer to search for set bits.
* @remark Similar to ffs() in BSD.
*/
{
unsigned long iBit;
iBit++;
else
iBit = 0;
"jnz 1f\n\t"
"xorl %0, %0\n\t"
"jmp 2f\n"
"1:\n\t"
"incl %0\n"
"2:\n\t"
: "=r" (iBit)
: "rm" (u32));
# else
{
done:
}
# endif
return iBit;
}
/**
* Finds the first bit which is set in the given 32-bit integer.
* Bits are numbered from 1 (least significant) to 32.
*
* @returns index [1..32] of the first set bit.
* @returns 0 if all bits are cleared.
* @param i32 Integer to search for set bits.
* @remark Similar to ffs() in BSD.
*/
{
}
/**
* Finds the last bit which is set in the given 32-bit integer.
* Bits are numbered from 1 (least significant) to 32.
*
* @returns index [1..32] of the last set bit.
* @returns 0 if all bits are cleared.
* @param u32 Integer to search for set bits.
* @remark Similar to fls() in BSD.
*/
{
unsigned long iBit;
iBit++;
else
iBit = 0;
"jnz 1f\n\t"
"xorl %0, %0\n\t"
"jmp 2f\n"
"1:\n\t"
"incl %0\n"
"2:\n\t"
: "=r" (iBit)
: "rm" (u32));
# else
{
done:
}
# endif
return iBit;
}
/**
* Finds the last bit which is set in the given 32-bit integer.
* Bits are numbered from 1 (least significant) to 32.
*
* @returns index [1..32] of the last set bit.
* @returns 0 if all bits are cleared.
* @param i32 Integer to search for set bits.
* @remark Similar to fls() in BSD.
*/
{
}
/**
* Reverse the byte order of the given 16-bit integer.
*
* @returns Revert
* @param u16 16-bit integer value.
*/
{
#else
{
}
#endif
return u16;
}
/**
* Reverse the byte order of the given 32-bit integer.
*
* @returns Revert
* @param u32 32-bit integer value.
*/
{
#else
{
}
#endif
return u32;
}
/**
* Reverse the byte order of the given 64-bit integer.
*
* @returns Revert
* @param u64 64-bit integer value.
*/
{
#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
#else
#endif
return u64;
}
/** @} */
/** @} */
#endif