0N/A; IPRT - Time using SUPLib, the Assembly Code Template.
0N/A; Copyright (C) 2006-2007 Sun Microsystems, Inc.
0N/A; This file is part of VirtualBox Open Source Edition (OSE), as
0N/A; you can redistribute it
and/or modify it under the terms of the GNU
0N/A; General Public License (GPL) as published by the Free Software
0N/A; Foundation, in version 2 as it comes in the "COPYING" file of the
0N/A; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
0N/A; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
0N/A; The contents of this file may alternatively be used under the terms
0N/A; of the Common Development and Distribution License Version 1.0
0N/A; VirtualBox OSE distribution, in which case the provisions of the
0N/A; CDDL are applicable instead of those of the GPL.
0N/A; You may elect to license modified versions of this file under the
0N/A; terms and conditions of either the GPL or the CDDL or both.
0N/A; Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
0N/A; additional information or have any questions.
0N/A; The x86 assembly implementation of the assembly routines.
0N/A; @returns Nanosecond timestamp.
0N/A; @param pData Pointer to the nanosecond timestamp data.
0N/ABEGINPROC rtTimeNanoTSInternalAsm
0N/A ; Variable definitions.
0N/A%define pData [ebp + 08h]
0N/A%define u64RetNanoTS_Hi [ebp - 04h]
0N/A%define u64RetNanoTS [ebp - 08h]
0N/A%define u32UpdateIntervalNS [ebp - 0ch]
0N/A%define u32UpdateIntervalTSC [ebp - 10h]
0N/A%define u64TSC_Hi [ebp - 14h]
0N/A%define u64TSC [ebp - 18h]
0N/A%define u64CurNanoTS_Hi [ebp - 1ch]
0N/A%define u64CurNanoTS [ebp - 20h]
0N/A%define u64PrevNanoTS_Hi [ebp - 24h]
0N/A%define u64PrevNanoTS [ebp - 28h]
0N/A%define u32TransactionId [ebp - 2ch]
0N/A%define u32ApicIdPlus [ebp - 30h]
0N/A%define TmpVar [ebp - 34h]
0N/A%define SavedEBX [ebp - 38h]
0N/A%define SavedEDI [ebp - 3ch]
0N/A%define SavedESI [ebp - 40h]
0N/A ;; Read the GIP data and the previous value.
0N/A ; Load pGip and calc pGipCPU, setting u32ApicIdPlus if necessary.
0N/A%ifdef IMPORTED_SUPLIB
0N/A mov esi, IMP(g_SUPGlobalInfoPage)
0N/A mov esi, IMP(g_pSUPGlobalInfoPage)
0N/A mov esi, [NAME(g_pSUPGlobalInfoPage)]
0N/A ; u8ApicId = ASMGetApicId();
0N/A %ifdef NEED_TRANSACTION_ID
0N/A mov u32ApicIdPlus, ebx
0N/A ; pGipCpu = &pGip->aCPU[u8ApicId];
0N/A mov eax, SUPGIPCPU_size
0N/A%ifdef NEED_TRANSACTION_ID
0N/A ; Serialized loading of u32TransactionId.
0N/A mov u32TransactionId, ebx
0N/A lock xor dword TmpVar, 0
0N/A ; Load the data and TSC.
0N/A mov u32UpdateIntervalNS, eax ; esi is now free
0N/A mov u32UpdateIntervalTSC, edx
0N/A mov u64CurNanoTS, ecx
0N/A mov u64CurNanoTS_Hi, esi
0N/A ; u64PrevNanoTS = ASMAtomicReadU64(pu64Prev);
0N/A ; RDTSC result, we try to make sure it has completed as well.
0N/A lock cmpxchg8b [esi]
0N/A mov u64PrevNanoTS, eax
0N/A mov u64PrevNanoTS_Hi, edx
0N/A%undef SAVED_u64RetNanoTS
0N/A%ifdef NEED_TRANSACTION_ID
0N/A ; Check that the GIP and CPU didn't change.
0N/A ; We've already serialized all the loads and stores at this point.
0N/A mov u64RetNanoTS, ebx
0N/A mov u64RetNanoTS_Hi, ecx
0N/A %define SAVED_u64RetNanoTS
0N/A cmp u32ApicIdPlus, ebx
0N/A cmp esi, u32TransactionId
0N/A%endif ; NEED_TRANSACTION_ID
0N/A%ifdef SAVED_u64RetNanoTS
0N/A mov ebx, u64RetNanoTS
0N/A mov ecx, u64RetNanoTS_Hi
0N/A ;; Calc the timestamp.
0N/A ; u64RetNanoTS -= u64TSC;
0N/A ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
0N/A cmp ebx, u32UpdateIntervalTSC
0N/A.ContinueCalcs: ; eax <= u32UpdateIntervalTSC
0N/A mul dword u32UpdateIntervalNS
0N/A div dword u32UpdateIntervalTSC
0N/A ; u64RetNanoTS += u64CurNanoTS;
0N/A add eax, u64CurNanoTS
0N/A adc edx, u64CurNanoTS_Hi
0N/A ;; Compare it with the previous one.
0N/A ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
0N/A ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
0N/A ;; @todo optimize this compare (/me too tired).
0N/A mov ecx, u64PrevNanoTS_Hi
0N/A mov ebx, u64PrevNanoTS
0N/A jbe .DeltaPrevTooBig
0N/A jae .DeltaPrevTooBig
0N/A ;; Update the previous value with the u64RetNanoTS value.
0N/A ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
0N/A mov eax, u64PrevNanoTS
0N/A mov edx, u64PrevNanoTS_Hi
0N/A lock cmpxchg8b [esi]
0N/A ;; We've expired the interval, cap it. If we're here for the 2nd
0N/A ;; time without any GIP update inbetween, the checks against
0N/A ;; pData->u64Prev below will force 1ns stepping.
0N/A ; u64Delta = u32UpdateIntervalTSC;
0N/A mov eax, u32UpdateIntervalTSC
0N/A ;; u64DeltaPrev >= 24h
38N/A ;; eax:edx = u64RetNanoTS (to be adjusted)
0N/A ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
0N/A sub ebx, u64PrevNanoTS
0N/A sbb ecx, u64PrevNanoTS_Hi ; ebx:ecx = u64DeltaPrev
0N/A ; else if ( (int64_t)u64DeltaPrev <= 0
0N/A ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
0N/A ; /* Occasional - u64RetNanoTS is in the recent 'past' relative the previous call. */
0N/A ; pData->c1nsSteps++;
0N/A ; u64RetNanoTS = u64PrevNanoTS + 1;
0N/A mov esi, u32UpdateIntervalNS
0N/A jl .PrevNotZero2ndTest
0N/A jg .DeltaPrevNotInRecentPast
0N/A ja .DeltaPrevNotInRecentPast
0N/A add esi, esi ; ASSUMES: u32UpdateIntervalNS * 2 <= 32-bit.
0N/A js .DeltaPrevNotInRecentPast
0N/A.DeltaPrevInRecentPast:
0N/A mov eax, u64PrevNanoTS
0N/A mov edx, u64PrevNanoTS_Hi
0N/A.DeltaPrevNotInRecentPast:
0N/A ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume). */
0N/A cmp dword u64PrevNanoTS, 0
0N/A jne .DeltaPrevNotZero
0N/A cmp dword u64PrevNanoTS_Hi, 0
0N/A ; /* Something has gone bust, if negative offset it's real bad. */
0N/A ; rtTimeNanoTSInternalBitch(pVM,
0N/A ; call C function that does the bitching.
0N/A mov u64RetNanoTS, eax
0N/A mov u64RetNanoTS_Hi, edx
0N/A mov edi, u64PrevNanoTS_Hi
0N/A push esi ; 4 - u64PrevNanoTS
0N/A push ebx ; 3 - u64DeltaPrev
0N/A push eax ; 2 - u64RetNanoTS
0N/A push eax ; 1 - pData
0N/A mov eax, u64RetNanoTS
0N/A mov edx, u64RetNanoTS_Hi
0N/A ;; Attempt updating the previous value, provided we're still ahead of it.
0N/A ;; There is no point in recalculating u64NanoTS because we got preemted or if
0N/A ;; we raced somebody while the GIP was updated, since these are events
0N/A ;; that might occure at any point in the return path as well.
0N/A ;; eax:edx = *pData->u64Prev
0N/A ;; ebx:ecx = u64RetNanoTS
0N/A ; for (i = 0; i < 10; i++)
0N/A ; if (u64PrevNanoTS >= u64NanoTS)
0N/A lock cmpxchg8b [esi]
0N/A ;; The GIP is seemingly invalid, redo the discovery.
0N/A%undef u32UpdateIntervalNS
0N/A%undef u32UpdateIntervalTSC
0N/A%undef u64PrevNanoTS_Hi
0N/A%undef u32TransactionId
0N/A; The AMD64 assembly implementation of the assembly routines.
0N/A; @returns Nanosecond timestamp.
0N/A; @param pData gcc:rdi msc:rcx Pointer to the nanosecond timestamp data.
0N/ABEGINPROC rtTimeNanoTSInternalAsm
0N/A ; Define variables and stack frame.
0N/A%define SavedRBX [rbp - 08h]
0N/A%define SavedR12 [rbp - 10h]
0N/A%define SavedR13 [rbp - 18h]
0N/A%define SavedRDI [rbp - 20h]
0N/A%define SavedRSI [rbp - 28h]
0N/A%define TmpVar [rbp - 30h]
0N/A%define TmpVar2 [rbp - 38h]
0N/A%ifdef NEED_TRANSACTION_ID
0N/A %define SavedR14 [rbp - 40h]
0N/A %define SavedR15 [rbp - 48h]
0N/A%define u32TransactionId r9d
0N/A%define u64CurNanoTS r10
0N/A%define u64PrevNanoTS r11 ; not parameter register
0N/A%define u32UpdateIntervalTSC r12d
0N/A%define u32UpdateIntervalTSC_64 r12
0N/A%define u32UpdateIntervalNS r13d
0N/A%define u32UpdateIntervalNS_64 r13
0N/A%undef u64SavedRetNanoTS
0N/A%ifdef NEED_TRANSACTION_ID
0N/A %define u64SavedRetNanoTS r14
0N/A %define u32ApicIdPlus r15d
0N/A%ifdef ASM_CALL64_MSC
0N/A%ifdef ASM_CALL64_MSC
0N/A ;mov pData, rdi - already in rdi.
0N/A ;; We take great pain ensuring that data consitency here.
0N/A ; Load pGip and calc pGipCPU, setting u32ApicIdPlus if necessary.
0N/A ; Finding the GIP is fun...
0N/A %ifdef IMPORTED_SUPLIB
0N/A mov rax, qword IMP(g_SUPGlobalInfoPage)
0N/A mov pGip, [IMP(g_pSUPGlobalInfoPage) wrt rip]
0N/A mov pGip, [NAME(g_pSUPGlobalInfoPage) wrt rip]
0N/A mov rax, qword NAME(g_SUPGlobalInfoPage)
0N/A ; u8ApicId = ASMGetApicId();
0N/A %ifdef NEED_TRANSACTION_ID
0N/A mov u32ApicIdPlus, ebx
0N/A ; pGipCpu = &pGip->aCPU[u8ApicId];
0N/A mov eax, SUPGIPCPU_size
0N/A%ifdef NEED_TRANSACTION_ID
0N/A ; Serialized loading of u32TransactionId.
859N/A lock xor dword TmpVar, 0
0N/A ; Load the data and TSC.
0N/A mov u64PrevNanoTS, [u64PrevNanoTS]
0N/A %ifdef u64SavedRetNanoTS ; doing this here saves a tick or so.
0N/A mov u64SavedRetNanoTS, rax
0N/A or u64SavedRetNanoTS, rdx
0N/A or rax, rdx ; rax is u64RetNanoTS.
0N/A%ifdef NEED_TRANSACTION_ID
0N/A ; Check that the GIP and CPU didn't change.
0N/A ; It is crucial that the rdtsc instruction has completed before
0N/A ; we check the transaction id. The LOCK prefixed instruction with
0N/A ; dependency on the RDTSC result should do the trick, I think.
0N/A ; CPUID is serializing, so the async path is safe by default.
38N/A cmp u32ApicIdPlus, ebx
0N/A lock xor qword TmpVar, rax
0N/A test u32TransactionId, 1
0N/A %ifdef u64SavedRetNanoTS
0N/A mov rax, u64SavedRetNanoTS ; rax is u64RetNanoTS.
0N/A%endif ; NEED_TRANSACTION_ID
0N/A ;; Calc the timestamp.
0N/A ; u64RetNanoTS -= u64TSC;
0N/A ; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
0N/A cmp rax, u32UpdateIntervalTSC_64
0N/A.ContinueCalcs: ; edx = 0; eax <= u32UpdateIntervalTSC
0N/A mul u32UpdateIntervalNS
0N/A div u32UpdateIntervalTSC
0N/A ; u64RetNanoTS += u64CurNanoTS;
0N/A add rax, u64CurNanoTS
0N/A ;; Compare it with the previous one.
0N/A ; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
0N/A ; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
0N/A ; /* Frequent - less than 24h since last call. */;
0N/A cmp rax, u64PrevNanoTS
0N/A jbe .DeltaPrevTooBig
0N/A shl rcx, 44 ; close enough
0N/A add rcx, u64PrevNanoTS
63N/A jae .DeltaPrevTooBig
63N/A ;; Update the previous value.
63N/A ; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
63N/A mov rax, u64PrevNanoTS
63N/A lock cmpxchg [rbx], rcx
0N/A%ifdef ASM_CALL64_MSC
0N/A ;; We've expired the interval, cap it. If we're here for the 2nd
0N/A ;; time without any GIP update inbetween, the checks against
0N/A ;; pData->u64Prev below will force 1ns stepping.
0N/A ; u64RetNanoTS = u32UpdateIntervalTSC;
0N/A mov eax, u32UpdateIntervalTSC
0N/A ;; u64DeltaPrev >= 24h
0N/A ;; rax = u64RetNanoTS (to be adjusted)
0N/A ; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
0N/A sub rbx, u64PrevNanoTS
0N/A ; else if ( (int64_t)u64DeltaPrev <= 0
0N/A ; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
0N/A ; /* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
0N/A ; pData->c1nsSteps++;
0N/A ; u64RetNanoTS = u64PrevNanoTS + 1;
0N/A jg .DeltaPrevNotInRecentPast
0N/A lea rdx, [u32UpdateIntervalNS_64 + u32UpdateIntervalNS_64]
0N/A js .DeltaPrevNotInRecentPast
0N/A lea rax, [u64PrevNanoTS + 1]
0N/A ; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume) / first call. */
0N/A.DeltaPrevNotInRecentPast:
0N/A or u64PrevNanoTS, u64PrevNanoTS
0N/A ; /* Something has gone bust, if negative offset it's real bad. */
0N/A ; rtTimeNanoTSInternalBitch(pVM,
0N/A ; call C function that does the bitching.
0N/A%ifdef ASM_CALL64_MSC
0N/A mov rcx, pData ; param 1 - pData
0N/A mov rdx, rax ; param 2 - u64RetNanoTS
0N/A mov r8, rbx ; param 3 - u64DeltaPrev
0N/A mov r9, u64PrevNanoTS ; param 4 - u64PrevNanoTS
0N/A ;mov rdi, pData - already in rdi; param 1 - pData
0N/A mov rsi, rax ; param 2 - u64RetNanoTS
0N/A mov rdx, rbx ; param 3 - u64DeltaPrev
0N/A mov rcx, u64PrevNanoTS ; param 4 - u64PrevNanoTS
0N/A ;; Attempt updating the previous value, provided we're still ahead of it.
0N/A ;; There is no point in recalculating u64NanoTS because we got preemted or if
0N/A ;; we raced somebody while the GIP was updated, since these are events
0N/A ;; that might occure at any point in the return path as well.
0N/A ;; rax = *pData->u64Prev;
0N/A ;; rcx = u64RetNanoTS
; for (i = 0; i < 10; i++)
; if (u64PrevNanoTS >= u64RetNanoTS)
;; The GIP is seemingly invalid, redo the discovery.
; mov rdi, pData - already in rdi
%undef u32UpdateIntervalTSC
%undef u32UpdateIntervalTSC_64
%undef u32UpdateIntervalNS
ENDPROC rtTimeNanoTSInternalAsm