SUPDrvGip.cpp revision 5acdcd39dfd0adec57d6e85838e999ba952ee49d
/* $Id$ */
/** @file
* VBoxDrv - The VirtualBox Support Driver - Common code for GIP.
*/
/*
* Copyright (C) 2006-2015 Oracle Corporation
*
* This file is part of VirtualBox Open Source Edition (OSE), as
* available from http://www.virtualbox.org. This file is free software;
* General Public License (GPL) as published by the Free Software
* Foundation, in version 2 as it comes in the "COPYING" file of the
* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
*
* The contents of this file may alternatively be used under the terms
* of the Common Development and Distribution License Version 1.0
* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
* VirtualBox OSE distribution, in which case the provisions of the
* CDDL are applicable instead of those of the GPL.
*
* You may elect to license modified versions of this file under the
* terms and conditions of either the GPL or the CDDL or both.
*/
/*******************************************************************************
* Header Files *
*******************************************************************************/
#define LOG_GROUP LOG_GROUP_SUP_DRV
#define SUPDRV_AGNOSTIC
#include "SUPDrvInternal.h"
#ifndef PAGE_SHIFT
#endif
#include <iprt/asm-amd64-x86.h>
#include <iprt/asm-math.h>
#include <iprt/handletable.h>
#include <iprt/semaphore.h>
#include <iprt/spinlock.h>
#endif
#if defined(RT_OS_SOLARIS) || defined(RT_OS_DARWIN)
#else
/* ... */
#endif
/*******************************************************************************
* Defined Constants And Macros *
*******************************************************************************/
/** The frequency by which we recalculate the u32UpdateHz and
* u32UpdateIntervalNS GIP members. The value must be a power of 2.
*
* Warning: Bumping this too high might overflow u32UpdateIntervalNS.
*/
#define GIP_UPDATEHZ_RECALC_FREQ 0x800
/** A reserved TSC value used for synchronization as well as measurement of
* TSC deltas. */
#define GIP_TSC_DELTA_RSVD UINT64_MAX
/** The number of TSC delta measurement loops in total (includes primer and
* read-time loops). */
#define GIP_TSC_DELTA_LOOPS 96
/** The number of cache primer loops. */
#define GIP_TSC_DELTA_PRIMER_LOOPS 4
/** The number of loops until we keep computing the minumum read time. */
#define GIP_TSC_DELTA_READ_TIME_LOOPS 24
/** The TSC frequency refinement period in seconds.
* The timer fires after 200ms, then every second, this value just says when
* to stop it after that. */
#define GIP_TSC_REFINE_PERIOD_IN_SECS 12
/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_PRACTICALLY_ZERO rating */
#define GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO 32
/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_ROUGHLY_ZERO rating */
#define GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO 448
/** The TSC delta value for the initial GIP master - 0 in regular builds.
* To test the delta code this can be set to a non-zero value. */
#if 0
#else
# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(0)
#endif
/** @def VBOX_SVN_REV
* The makefile should define this if it can. */
#ifndef VBOX_SVN_REV
# define VBOX_SVN_REV 0
#endif
#if 0 /* Don't start the GIP timers. Useful when debugging the IPRT timer code. */
# define DO_NOT_START_GIP
#endif
/*******************************************************************************
* Internal Functions *
*******************************************************************************/
static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
static void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz);
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
#else
#endif
/*******************************************************************************
* Global Variables *
*******************************************************************************/
/*
*
* Misc Common GIP Code
* Misc Common GIP Code
* Misc Common GIP Code
*
*
*/
/**
* Finds the GIP CPU index corresponding to @a idCpu.
*
* @returns GIP CPU array index, UINT32_MAX if not found.
* @param pGip The GIP.
* @param idCpu The CPU ID.
*/
{
uint32_t i;
return i;
return UINT32_MAX;
}
/*
*
* GIP Mapping and Unmapping Related Code.
* GIP Mapping and Unmapping Related Code.
* GIP Mapping and Unmapping Related Code.
*
*
*/
/**
* (Re-)initializes the per-cpu structure prior to starting or resuming the GIP
* updating.
*
* @param pGip Pointer to the GIP.
* @param pGipCpu The per CPU structure for this CPU.
* @param u64NanoTS The current time.
*/
{
/*
* Here we don't really care about applying the TSC delta. The re-initialization of this
* value is not relevant especially while (re)starting the GIP as the first few ones will
* be ignored anyway, see supdrvGipDoUpdateCpu().
*/
}
/**
* Set the current TSC and NanoTS value for the CPU.
*
* @param idCpu The CPU ID. Unused - we have to use the APIC ID.
* @param pvUser1 Pointer to the ring-0 GIP mapping.
* @param pvUser2 Pointer to the variable holding the current time.
*/
{
}
/**
* State structure for supdrvGipDetectGetGipCpuCallback.
*/
typedef struct SUPDRVGIPDETECTGETCPU
{
/** Bitmap of APIC IDs that has been seen (initialized to zero).
* Used to detect duplicate APIC IDs (paranoia). */
/** Mask of supported GIP CPU getter methods (SUPGIPGETCPU_XXX) (all bits set
* initially). The callback clears the methods not detected. */
uint32_t volatile fSupported;
/** The first callback detecting any kind of range issues (initialized to
* NIL_RTCPUID). */
RTCPUID volatile idCpuProblem;
/** Pointer to state structure for supdrvGipDetectGetGipCpuCallback. */
typedef SUPDRVGIPDETECTGETCPU *PSUPDRVGIPDETECTGETCPU;
/**
* Checks for alternative ways of getting the CPU ID.
*
* This also checks the APIC ID, CPU ID and CPU set index values against the
* GIP tables.
*
* @param idCpu The CPU ID. Unused - we have to use the APIC ID.
* @param pvUser1 Pointer to the state structure.
* @param pvUser2 Pointer to the GIP.
*/
static DECLCALLBACK(void) supdrvGipDetectGetGipCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
{
uint32_t fSupported = 0;
int iCpuSet;
AssertMsg(idCpu == RTMpCpuId(), ("idCpu=%#x RTMpCpuId()=%#x\n", idCpu, RTMpCpuId())); /* paranoia^3 */
/*
* Check that the CPU ID and CPU set index are interchangable.
*/
{
if ( iCpuSet >= 0
&& iCpuSet < RTCPUSET_MAX_CPUS
{
/*
* Check whether the IDTR.LIMIT contains a CPU number.
*/
#ifdef RT_ARCH_X86
#else
#endif
ASMGetIDTR(&Idtr);
{
{
ASMGetIDTR(&Idtr2);
}
}
/*
* Check whether RDTSCP is an option.
*/
if (ASMHasCpuId())
{
{
{
ASMNopPause();
}
}
}
}
}
/*
* Check that the APIC ID is unique.
*/
idApic = ASMGetApicId();
else
{
LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - duplicate APIC ID.\n",
}
/*
* Check that the iCpuSet is within the expected range.
*/
if (RT_UNLIKELY( iCpuSet < 0
|| (unsigned)iCpuSet >= RTCPUSET_MAX_CPUS
{
LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU set index is out of range.\n",
}
else
{
{
LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU id/index roundtrip problem: %#x\n",
}
}
/*
* Update the supported feature mask before we return.
*/
}
/**
* Increase the timer freqency on hosts where this is possible (NT).
*
* The idea is that more interrupts is better for us... Also, it's better than
* we increase the timer frequence, because we might end up getting inaccurate
* callbacks if someone else does it.
*
* @param pDevExt Sets u32SystemTimerGranularityGrant if increased.
*/
{
if (pDevExt->u32SystemTimerGranularityGrant == 0)
{
)
{
}
}
}
/**
* Undoes supdrvGipRequestHigherTimerFrequencyFromSystem.
*
* @param pDevExt Clears u32SystemTimerGranularityGrant.
*/
{
{
}
}
/**
*
* @returns IPRT status code.
* @param pSession Session to which the GIP mapping should belong.
* @param ppGipR3 Where to store the address of the ring-3 mapping. (optional)
* @param pHCPhysGip Where to store the physical address. (optional)
*
* @remark There is no reference counting on the mapping, so one call to this function
* count globally as one reference. One call to SUPR0GipUnmap() is will unmap GIP
* and remove the session as a GIP user.
*/
{
int rc;
/*
* Validate
*/
#ifdef SUPDRV_USE_MUTEX_FOR_GIP
#else
#endif
{
/*
* Map it?
*/
rc = VINF_SUCCESS;
if (ppGipR3)
{
if (RT_SUCCESS(rc))
}
/*
* Get physical address.
*/
/*
* Reference globally.
*/
{
{
/*
* host timer frequency to make sure we don't get stuck in guest
* mode and to get better timer (and possibly clock) accuracy.
*/
LogFlow(("SUPR0GipMap: Resumes GIP updating\n"));
/*
* document me
*/
{
unsigned i;
}
/*
* document me
*/
|| RTMpGetOnlineCount() == 1)
else
/*
* Detect alternative ways to figure the CPU ID in ring-3 and
* raw-mode context. Check the sanity of the APIC IDs, CPU IDs,
* and CPU set indexes while we're at it.
*/
if (RT_SUCCESS(rc))
{
{
&& DetectState.fSupported != 0)
{
{
}
}
else
{
LogRel(("SUPR0GipMap: No supported ways of getting the APIC ID or CPU number in ring-3! (%#x)\n",
}
}
else
{
LogRel(("SUPR0GipMap: APIC ID, CPU ID or CPU set index problem detected on CPU #%u (%#x)!\n",
}
}
/*
* Start the GIP timer if all is well..
*/
if (RT_SUCCESS(rc))
{
#ifndef DO_NOT_START_GIP
#endif
rc = VINF_SUCCESS;
}
/*
* Bail out on error.
*/
if (RT_FAILURE(rc))
{
pSession->fGipReferenced = 0;
{
if (RT_SUCCESS(rc2))
}
}
}
}
}
else
{
Log(("SUPR0GipMap: GIP is not available!\n"));
}
#ifdef SUPDRV_USE_MUTEX_FOR_GIP
#else
#endif
/*
* Write returns.
*/
if (pHCPhysGip)
*pHCPhysGip = HCPhys;
if (ppGipR3)
#ifdef DEBUG_DARWIN_GIP
OSDBGPRINT(("SUPR0GipMap: returns %d *pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void *)pGipR3));
#else
LogFlow(( "SUPR0GipMap: returns %d *pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void *)pGipR3));
#endif
return rc;
}
/**
* Unmaps any user mapping of the GIP and terminates all GIP access
* from this session.
*
* @returns IPRT status code.
* @param pSession Session to which the GIP mapping should belong.
*/
{
int rc = VINF_SUCCESS;
#ifdef DEBUG_DARWIN_GIP
OSDBGPRINT(("SUPR0GipUnmap: pSession=%p pGip=%p GipMapObjR3=%p\n",
pSession->GipMapObjR3));
#else
#endif
#ifdef SUPDRV_USE_MUTEX_FOR_GIP
#else
#endif
/*
* Unmap anything?
*/
{
if (RT_SUCCESS(rc))
}
/*
* Dereference global GIP.
*/
{
pSession->fGipReferenced = 0;
{
LogFlow(("SUPR0GipUnmap: Suspends GIP updating\n"));
#ifndef DO_NOT_START_GIP
#endif
}
}
#ifdef SUPDRV_USE_MUTEX_FOR_GIP
#else
#endif
return rc;
}
/**
* Gets the GIP pointer.
*
* @returns Pointer to the GIP or NULL.
*/
{
return g_pSUPGlobalInfoPage;
}
/*
*
*
* GIP Initialization, Termination and CPU Offline / Online Related Code.
* GIP Initialization, Termination and CPU Offline / Online Related Code.
* GIP Initialization, Termination and CPU Offline / Online Related Code.
*
*
*/
/**
* Used by supdrvInitRefineInvariantTscFreqTimer and supdrvGipInitMeasureTscFreq
* to update the TSC frequency related GIP variables.
*
* @param pGip The GIP.
* @param nsElapsed The number of nano seconds elapsed.
* @param cElapsedTscTicks The corresponding number of TSC ticks.
* @param iTick The tick number for debugging.
*/
static void supdrvGipInitSetCpuFreq(PSUPGLOBALINFOPAGE pGip, uint64_t nsElapsed, uint64_t cElapsedTscTicks, uint32_t iTick)
{
/*
* Calculate the frequency.
*/
&& nsElapsed < UINT32_MAX)
else
{
}
/*
* Update the GIP.
*/
{
/* For inspecting the frequency calcs using tstGIP-2, debugger or similar. */
}
}
/**
* Timer callback function for TSC frequency refinement in invariant GIP mode.
*
* This is started during driver init and fires once
* GIP_TSC_REFINE_PERIOD_IN_SECS seconds later.
*
* @param pTimer The timer.
* @param pvUser Opaque pointer to the device instance data.
* @param iTick The timer tick.
*/
static DECLCALLBACK(void) supdrvInitRefineInvariantTscFreqTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
{
/* Paranoia. */
/*
* If we got a power event, stop the refinement process.
*/
{
return;
}
/*
* Read the TSC and time, noting which CPU we are on.
*
* Don't bother spinning until RTTimeSystemNanoTS changes, since on
* systems where it matters we're in a context where we cannot waste that
* much time (DPC watchdog, called from clock interrupt).
*/
uTsc = ASMReadTSC();
nsNow = RTTimeSystemNanoTS();
/*
* If the above measurement was taken on a different CPU than the one we
* started the process on, cTscTicksElapsed will need to be adjusted with
* the TSC deltas of both the CPUs.
*
* We ASSUME that the delta calculation process takes less time than the
* TSC frequency refinement timer. If it doesn't, we'll complain and
* drop the frequency refinement.
*
* Note! We cannot entirely trust enmUseTscDelta here because it's
* downgraded after each delta calculation.
*/
{
int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
{
{
/* cTscTicksElapsed = (uTsc - iStopTscDelta) - (pDevExt->uTscStartInvarTscRefine - iStartTscDelta); */
}
}
/*
* Allow 5 times the refinement period to elapse before we give up on the TSC delta
* calculations.
*/
{
SUPR0Printf("vboxdrv: Failed to refine invariant TSC frequency because deltas are unavailable after %u (%u) seconds\n",
SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
return;
}
}
/*
* Calculate and update the CPU frequency variables in GIP.
*
* If there is a GIP user already and we've already refined the frequency
* a couple of times, don't update it as we want a stable frequency value
* for all VMs.
*/
{
/*
* Stop the timer once we've reached the defined refinement period.
*/
{
}
}
else
{
}
}
/**
* @callback_method_impl{FNRTPOWERNOTIFICATION}
*/
{
/*
* If the TSC frequency refinement timer we need to cancel it so it doesn't screw
* up the frequency after a long suspend.
*/
if ( enmEvent == RTPOWEREVENT_SUSPEND
|| enmEvent == RTPOWEREVENT_RESUME)
}
/**
* Start the TSC-frequency refinment timer for the invariant TSC GIP mode.
*
* We cannot use this in the synchronous and asynchronous tsc GIP modes because
* the CPU may change the TSC frequence between now and when the timer fires
* (supdrvInitAsyncRefineTscTimer).
*
* @param pDevExt Pointer to the device instance data.
* @param pGip Pointer to the GIP.
*/
static void supdrvGipInitStartTimerForRefiningInvariantTscFreq(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip)
{
int rc;
/*
* Register a power management callback.
*/
pDevExt->fInvTscRefinePowerEvent = false;
/*
* Record the TSC and NanoTS as the starting anchor point for refinement
* of the TSC. We try get as close to a clock tick as possible on systems
* which does not provide high resolution time.
*/
while (RTTimeSystemNanoTS() == u64NanoTS)
ASMNopPause();
/*
* Create a timer that runs on the same CPU so we won't have a depencency
* on the TSC-delta and can run in parallel to it. On systems that does not
* implement CPU specific timers we'll apply deltas in the timer callback,
* just like we do for CPUs going offline.
*
* The longer the refinement interval the better the accuracy, at least in
* theory. If it's too long though, ring-3 may already be starting its
* first VMs before we're done. On most systems we will be loading the
* support driver during boot and VMs won't be started for a while yet,
* it is really only a problem during development (especially with
* on-demand driver starting on windows).
*
* To avoid wasting time doing a long supdrvGipInitMeasureTscFreq() call
* to calculate the frequency during driver loading, the timer is set
* to fire after 200 ms the first time. It will then reschedule itself
* to fire every second until GIP_TSC_REFINE_PERIOD_IN_SECS has been
* reached or it notices that there is a user land client with GIP
* mapped (we want a stable frequency for all VMs).
*/
if (RT_SUCCESS(rc))
{
if (RT_SUCCESS(rc))
return;
}
{
if (RT_SUCCESS(rc))
{
if (RT_SUCCESS(rc))
return;
}
}
}
/**
* @callback_method_impl{PFNRTMPWORKER,
* RTMpOnSpecific callback for reading TSC and time on the CPU we started
* the measurements on.}
*/
{
*puTscStop = ASMReadTSC();
*pnsStop = RTTimeSystemNanoTS();
}
/**
* Measures the TSC frequency of the system.
*
* The TSC frequency can vary on systems which are not reported as invariant.
* On such systems the object of this function is to find out what the nominal,
* maximum TSC frequency under 'normal' CPU operation.
*
* @returns VBox status code.
* @param pDevExt Pointer to the device instance.
* @param pGip Pointer to the GIP.
* @param fRough Set if we're doing the rough calculation that the
* TSC measuring code needs, where accuracy isn't all
* that important (too high is better than to low).
* When clear we try for best accuracy that we can
* achieve in reasonably short time.
*/
{
while (cTriesLeft-- > 0)
{
/*
* Synchronize with the host OS clock tick on systems without high
* resolution time API (older Windows version for example).
*/
while (RTTimeSystemNanoTS() == nsStart)
ASMNopPause();
/*
* Read the TSC and current time, noting which CPU we're on.
*/
uTscStart = ASMReadTSC();
idCpuStart = RTMpCpuId();
/*
* Delay for a while.
*/
{
/*
* Sleep-wait since the TSC frequency is constant, it eases host load.
* Shorter interval produces more variance in the frequency (esp. Windows).
*/
uint64_t msDelay = ( ((fRough ? 16 : 200) * RT_NS_1MS + nsTimerIncr - 1) / nsTimerIncr * nsTimerIncr - RT_NS_100US )
/ RT_NS_1MS;
do
{
nsStop = RTTimeSystemNanoTS();
while (RTTimeSystemNanoTS() == nsStop)
ASMNopPause();
}
else
{
/*
* Busy-wait keeping the frequency up.
*/
do
{
ASMNopPause();
nsStop = RTTimeSystemNanoTS();
}
/*
* Read the TSC and time again.
*/
uTscStop = ASMReadTSC();
nsStop = RTTimeSystemNanoTS();
/*
* If the CPU changes things get a bit complicated and what we
* can get away with depends on the GIP mode / TSC reliablity.
*/
if (idCpuStop != idCpuStart)
{
bool fDoXCall = false;
/*
* Synchronous TSC mode: we're probably fine as it's unlikely
* that we were rescheduled because of TSC throttling or power
* management reasons, so just go ahead.
*/
{
/* Probably ok, maybe we should retry once?. */
}
/*
* If we're just doing the rough measurement, do the cross call and
* get on with things (we don't have deltas!).
*/
else if (fRough)
fDoXCall = true;
/*
* Invariant TSC mode: It doesn't matter if we have delta available
* for both CPUs. That is not something we can assume at this point.
*
* Note! We cannot necessarily trust enmUseTscDelta here because it's
* downgraded after each delta calculation and the delta
* calculations may not be complete yet.
*/
{
/** @todo This section of code is never reached atm, consider dropping it later on... */
{
int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
{
{
}
}
/*
* we have to trigger driver load failure if that happens as GIP
* and IPRT assumptions are busted on this system.
*/
{
SUPR0Printf("vboxdrv: Unexpected CPU index in supdrvGipInitMeasureTscFreq.\n");
SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
return VERR_INVALID_CPU_INDEX;
}
/*
* No valid deltas. We retry, if we're on our last retry
* we do the cross call instead just to get a result. The
* frequency will be refined in a few seconds anyways.
*/
else if (cTriesLeft > 0)
continue;
else
fDoXCall = true;
}
}
/*
* Asynchronous TSC mode: This is bad as the reason we usually
* use this mode is to deal with variable TSC frequencies and
* deltas. So, we need to get the TSC from the same CPU as
* started it, we also need to keep that CPU busy. So, retry
* and fall back to the cross call on the last attempt.
*/
else
{
if (cTriesLeft > 0)
continue;
fDoXCall = true;
}
if (fDoXCall)
{
/*
* Try read the TSC and timestamp on the start CPU.
*/
continue;
}
}
/*
* Calculate the TSC frequency and update it (shared with the refinement timer).
*/
return VINF_SUCCESS;
}
}
/**
* Finds our (@a idCpu) entry, or allocates a new one if not found.
*
* @returns Index of the CPU in the cache set.
* @param pGip The GIP.
* @param idCpu The CPU ID.
*/
{
/*
* ASSUMES that CPU IDs are constant.
*/
return i;
cTries = 0;
do
{
{
bool fRc;
if (fRc)
return i;
}
} while (cTries++ < 32);
return i - 1;
}
/**
* The calling CPU should be accounted as online, update GIP accordingly.
*
* This is used by supdrvGipCreate() as well as supdrvGipMpEvent().
*
* @param pDevExt The device extension.
* @param idCpu The CPU ID.
*/
{
int iCpuSet = 0;
uint32_t i = 0;
/*
* Do this behind a spinlock with interrupts disabled as this can fire
* on all CPUs simultaneously, see @bugref{6110}.
*/
/*
* Update the globals.
*/
if (iCpuSet >= 0)
{
}
/*
* Update the entry.
*/
idApic = ASMGetApicId();
/*
* Update the APIC ID and CPU set index mappings.
*/
/* Add this CPU to this set of CPUs we need to calculate the TSC-delta for. */
/* Commit it. */
}
/**
* RTMpOnSpecific callback wrapper for supdrvGipMpEventOnlineOrInitOnCpu().
*
* @param idCpu The CPU ID we are running on.
* @param pvUser1 Opaque pointer to the device instance data.
* @param pvUser2 Not used.
*/
static DECLCALLBACK(void) supdrvGipMpEventOnlineCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
{
}
/**
* The CPU should be accounted as offline, update the GIP accordingly.
*
* This is used by supdrvGipMpEvent.
*
* @param pDevExt The device extension.
* @param idCpu The CPU ID.
*/
{
int iCpuSet;
unsigned i;
AssertReturnVoid(iCpuSet >= 0);
{
/* Reset the TSC delta, we will recalculate it lazily. */
/* Remove this CPU from the set of CPUs that we have obtained the TSC deltas. */
}
/* Commit it. */
}
/**
* Multiprocessor event notification callback.
*
* This is used to make sure that the GIP master gets passed on to
* another CPU. It also updates the associated CPU data.
*
* @param enmEvent The event.
* @param idCpu The cpu it applies to.
* @param pvUser Pointer to the device extension.
*/
{
if (pGip)
{
switch (enmEvent)
{
case RTMPEVENT_ONLINE:
{
{
}
else
{
}
/*
* Recompute TSC-delta for the newly online'd CPU.
*/
{
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
#else
#endif
}
break;
}
case RTMPEVENT_OFFLINE:
break;
}
}
/*
* Make sure there is a master GIP.
*/
if (enmEvent == RTMPEVENT_OFFLINE)
{
if (idGipMaster == idCpu)
{
/*
* The GIP master is going offline, find a new one.
*/
bool fIgnored;
unsigned i;
for (i = 0; i < RTCPUSET_MAX_CPUS; i++)
if (RTCpuSetIsMemberByIndex(&OnlineCpus, i))
{
if (idCurCpu != idGipMaster)
{
break;
}
}
}
}
}
/**
* On CPU initialization callback for RTMpOnAll.
*
* @param idCpu The CPU ID.
* @param pvUser1 The device extension.
* @param pvUser2 The GIP.
*/
{
/* This is good enough, even though it will update some of the globals a
bit to much. */
}
/**
* Callback used by supdrvDetermineAsyncTSC to read the TSC on a CPU.
*
* @param idCpu Ignored.
* @param pvUser1 Where to put the TSC.
* @param pvUser2 Ignored.
*/
static DECLCALLBACK(void) supdrvGipInitDetermineAsyncTscWorker(RTCPUID idCpu, void *pvUser1, void *pvUser2)
{
}
/**
* Determine if Async GIP mode is required because of TSC drift.
*
* (TSC) runs never backwards, that is, a read operation to the counter should return
* a bigger value than any previous read operation. This is guaranteed by the latest
* AMD CPUs and by newer Intel CPUs which never enter the C2 state (P4). In any other
* case we have to choose the asynchronous timer mode.
*
* @param poffMin Pointer to the determined difference between different
* cores (optional, can be NULL).
* @return false if the time stamp counters appear to be synchronized, true otherwise.
*/
{
/*
* Just iterate all the cpus 8 times and make sure that the TSC is
* ever increasing. We don't bother taking TSC rollover into account.
*/
int iEndCpu = RTMpGetArraySize();
int iCpu;
int cLoops = 8;
bool fAsync = false;
int rc = VINF_SUCCESS;
while (cLoops-- > 0)
{
{
if (RT_SUCCESS(rc))
{
{
fAsync = true;
Log(("supdrvGipInitDetermineAsyncTsc: iCpu=%d cLoops=%d CurTsc=%llx PrevTsc=%llx\n",
break;
}
/* Gather statistics (except the first time). */
{
}
/* Next */
}
else if (rc == VERR_NOT_SUPPORTED)
break;
else
}
/* broke out of the loop. */
break;
}
if (poffMin)
Log(("supdrvGipInitDetermineAsyncTsc: returns %d; iEndCpu=%d rc=%d offMin=%llx offMax=%llx\n",
#endif
return fAsync;
}
/**
* supdrvGipInit() worker that determines the GIP TSC mode.
*
* @returns The most suitable TSC mode.
* @param pDevExt Pointer to the device instance data.
*/
{
/*
* Establish whether the CPU advertises TSC as invariant, we need that in
* a couple of places below.
*/
bool fInvariantTsc = false;
if (ASMHasCpuId())
{
{
fInvariantTsc = true;
}
}
/*
* On single CPU systems, we don't need to consider ASYNC mode.
*/
if (RTMpGetCount() <= 1)
/*
*/
return SUPGIPMODE_ASYNC_TSC;
/*
* Use invariant mode if the CPU says TSC is invariant.
*/
if (fInvariantTsc)
return SUPGIPMODE_INVARIANT_TSC;
/*
* TSC is not invariant and we're on SMP, this presents two problems:
*
* (1) There might be a skew between the CPU, so that cpu0
* returns a TSC that is slightly different from cpu1.
* This screw may be due to (2), bad TSC initialization
* or slightly different TSC rates.
*
* (2) Power management (and other things) may cause the TSC
* to run at a non-constant speed, and cause the speed
* to be different on the cpus. This will result in (1).
*
* If any of the above is detected, we will have to use ASYNC mode.
*/
/* (1). Try check for current differences between the cpus. */
return SUPGIPMODE_ASYNC_TSC;
/* (2) If it's an AMD CPU with power management, we won't trust its TSC. */
if ( ASMIsValidStdRange(uEAX)
{
/* Check for APM support. */
{
return SUPGIPMODE_ASYNC_TSC;
}
}
return SUPGIPMODE_SYNC_TSC;
}
/**
* Initializes per-CPU GIP information.
*
* @param pGip Pointer to the GIP.
* @param pCpu Pointer to which GIP CPU to initalize.
* @param u64NanoTS The current nanosecond timestamp.
* @param uCpuHz The CPU frequency to set, 0 if the caller doesn't know.
*/
static void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz)
{
/*
* The first time we're called, we don't have a CPU frequency handy,
* so pretend it's a 4 GHz CPU. On CPUs that are online, we'll get
* called again and at that point we have a more plausible CPU frequency
* value handy. The frequency history will also be adjusted again on
* the 2nd timer callout (maybe we can skip that now?).
*/
if (!uCpuHz)
{
}
else
{
}
pCpu->au32TSCHistory[0]
}
/**
* Initializes the GIP data.
*
* @param pDevExt Pointer to the device instance data.
* @param pGip Pointer to the read-write kernel mapping of the GIP.
* @param HCPhys The physical address of the GIP.
* @param u64NanoTS The current nanosecond timestamp.
* @param uUpdateHz The update frequency.
* @param uUpdateIntervalNS The update interval in nanoseconds.
* @param cCpus The CPU count.
*/
{
unsigned i;
#ifdef DEBUG_DARWIN_GIP
OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
#else
LogFlow(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
#endif
/*
* Initialize the structure.
*/
/*|| pGip->u32Mode == SUPGIPMODE_SYNC_TSC */)
else
for (i = 0; i < cCpus; i++)
/*
* Link it to the device extension.
*/
}
/**
* Creates the GIP.
*
* @returns VBox status code.
* @param pDevExt Instance data. GIP stuff may be updated.
*/
{
unsigned cCpus;
int rc;
LogFlow(("supdrvGipCreate:\n"));
/*
* Assert order.
*/
#ifdef SUPDRV_USE_MUTEX_FOR_GIP
#else
#endif
/*
* Check the CPU count.
*/
cCpus = RTMpGetArraySize();
if ( cCpus > RTCPUSET_MAX_CPUS
{
SUPR0Printf("VBoxDrv: Too many CPUs (%u) for the GIP (max %u)\n", cCpus, RT_MIN(RTCPUSET_MAX_CPUS, 256));
return VERR_TOO_MANY_CPUS;
}
/*
* Allocate a contiguous set of pages with a default kernel mapping.
*/
rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), false /*fExecutable*/);
if (RT_FAILURE(rc))
{
return rc;
}
/*
* Find a reasonable update interval and initialize the structure.
*/
/** @todo figure out why using a 100Ms interval upsets timekeeping in VMs.
* See @bugref{6710}. */
if (uMod)
supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /*=Hz*/, u32Interval, cCpus);
/*
* Important sanity check...
*/
{
/* Basically, invariant Windows boxes, should never be detected as async (i.e. TSC-deltas should be 0). */
OSDBGPRINT(("supdrvGipCreate: The TSC-deltas should be normalized by the host OS, but verifying shows it's not!\n"));
return VERR_INTERNAL_ERROR_2;
}
/*
* Do the TSC frequency measurements.
*
* If we're in invariant TSC mode, just to a quick preliminary measurement
* that the TSC-delta measurement code can use to yield cross calls.
*
* If we're in any of the other two modes, neither which require MP init,
* notifications or deltas for the job, do the full measurement now so
* that supdrvGipInitOnCpu() can populate the TSC interval and history
* array with more reasonable values.
*/
{
}
else
if (RT_SUCCESS(rc))
{
/*
* Start TSC-delta measurement thread before we start getting MP
* events that will try kick it into action (includes the
* RTMpOnAll/supdrvGipInitOnCpu call below).
*/
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
#endif
if (RT_SUCCESS(rc))
{
if (RT_SUCCESS(rc))
{
/*
* Do GIP initialization on all online CPUs. Wake up the
* TSC-delta thread afterwards.
*/
if (RT_SUCCESS(rc))
{
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
#else
{
/*
* Measure the TSC deltas now that we have MP notifications.
*/
int cTries = 5;
do
{
if ( rc != VERR_TRY_AGAIN
&& rc != VERR_CPU_OFFLINE)
break;
} while (--cTries > 0);
}
else
{
AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode));
}
if (RT_SUCCESS(rc))
#endif
{
/*
* Create the timer.
* If CPU_ALL isn't supported we'll have to fall back to synchronous mode.
*/
{
if (rc == VERR_NOT_SUPPORTED)
{
OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n"));
}
}
if (RT_SUCCESS(rc))
{
/*
* We're good.
*/
return VINF_SUCCESS;
}
OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc));
}
}
else
}
else
}
else
}
else
/* Releases timer frequency increase too. */
return rc;
}
/**
* Invalidates the GIP data upon termination.
*
* @param pGip Pointer to the read-write kernel mapping of the GIP.
*/
{
unsigned i;
{
}
}
/**
* Terminates the GIP.
*
* @param pDevExt Instance data. GIP stuff may be updated.
*/
{
int rc;
#ifdef DEBUG_DARWIN_GIP
#endif
/*
* Stop receiving MP notifications before tearing anything else down.
*/
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
/*
* Terminate the TSC-delta measurement thread and resources.
*/
#endif
/*
* Destroy the TSC-refinement timer.
*/
if (pDevExt->pInvarTscRefineTimer)
{
}
/*
* Invalid the GIP data.
*/
{
}
/*
* Destroy the timer and free the GIP memory object.
*/
{
}
{
}
/*
* Finally, make sure we've release the system timer resolution request
* if one actually succeeded and is still pending.
*/
}
/*
*
*
* GIP Update Timer Related Code
* GIP Update Timer Related Code
* GIP Update Timer Related Code
*
*
*/
/**
* Worker routine for supdrvGipUpdate() and supdrvGipUpdatePerCpu() that
* updates all the per cpu data except the transaction id.
*
* @param pDevExt The device extension.
* @param pGipCpu Pointer to the per cpu data.
* @param u64NanoTS The current time stamp.
* @param u64TSC The current TSC.
* @param iTick The current timer tick.
*
* @remarks Can be called with interrupts disabled!
*/
static void supdrvGipDoUpdateCpu(PSUPDRVDEVEXT pDevExt, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS, uint64_t u64TSC, uint64_t iTick)
{
unsigned iTSCHistoryHead;
/* Delta between this and the previous update. */
/*
* Update the NanoTS.
*/
/*
* Calc TSC delta.
*/
/*
* We don't need to keep realculating the frequency when it's invariant, so
* the remainder of this function is only for the sync and async TSC modes.
*/
{
if (u64TSCDelta >> 32)
{
}
/*
* On the 2nd and 3rd callout, reset the history with the current TSC
* interval since the values entered by supdrvGipInit are totally off.
* The interval on the 1st callout completely unreliable, the 2nd is a bit
* better, while the 3rd should be most reliable.
*/
/** @todo Could we drop this now that we initializes the history
* with nominal TSC frequency values? */
|| u32TransactionId == 7)
&& ( iTick == 2
|| iTick == 3) ))
{
unsigned i;
}
/*
* Validate the NanoTS deltas between timer fires with an arbitrary threshold of 0.5%.
* Wait until we have at least one full history since the above history reset. The
* assumption is that the majority of the previous history values will be tolerable.
* See @bugref{6710} comment #67.
*/
/** @todo Could we drop the fuding there now that we initializes the history
* with nominal TSC frequency values? */
{
{
u32 >>= 2;
u64TSCDelta >>= 2;
u64TSCDelta += u32;
u64TSCDelta >>= 1;
}
}
/*
* TSC History.
*/
/*
* UpdateIntervalTSC = average of last 8,2,1 intervals depending on update HZ.
*
* On Windows, we have an occasional (but recurring) sour value that messed up
* the history but taking only 1 interval reduces the precision overall.
*/
{
u32 >>= 2;
u32UpdateIntervalTSC >>= 2;
u32UpdateIntervalTSC >>= 1;
/* Value chosen for a 2GHz Athlon64 running linux 2.6.10/11. */
}
{
u32UpdateIntervalTSC >>= 1;
/* value chosen on a 2GHz thinkpad running windows */
}
else
{
/* This value hasn't be checked yet.. waiting for OS/2 and 33Hz timers.. :-) */
}
ASMAtomicWriteU32(&pGipCpu->u32UpdateIntervalTSC, u32UpdateIntervalTSC + u32UpdateIntervalTSCSlack);
/*
* CpuHz.
*/
}
}
/**
* Updates the GIP.
*
* @param pDevExt The device extension.
* @param u64NanoTS The current nanosecond timesamp.
* @param u64TSC The current TSC timesamp.
* @param idCpu The CPU ID.
* @param iTick The current timer tick.
*
* @remarks Can be called with interrupts disabled!
*/
static void supdrvGipUpdate(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC, RTCPUID idCpu, uint64_t iTick)
{
/*
* Determine the relevant CPU data.
*/
else
{
return;
return;
}
/*
* Start update transaction.
*/
{
/* this can happen on win32 if we're taking to long and there are more CPUs around. shouldn't happen though. */
return;
}
/*
* Recalc the update frequency every 0x800th time.
*/
if ( pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC /* cuz we're not recalculating the frequency on invariants hosts. */
{
if (pGip->u64NanoTSLastUpdateHz)
{
#ifdef RT_ARCH_AMD64 /** @todo fix 64-bit div here to work on x86 linux. */
{
/** @todo r=ramshankar: Changing u32UpdateHz might screw up TSC frequency
* calculation on non-invariant hosts if it changes the history decision
* taken in supdrvGipDoUpdateCpu(). */
}
#endif
}
}
/*
* Update the data.
*/
/*
* Complete transaction.
*/
}
/**
* Updates the per cpu GIP data for the calling cpu.
*
* @param pDevExt The device extension.
* @param u64NanoTS The current nanosecond timesamp.
* @param u64TSC The current TSC timesamp.
* @param idCpu The CPU ID.
* @param idApic The APIC id for the CPU index.
* @param iTick The current timer tick.
*
* @remarks Can be called with interrupts disabled!
*/
{
/*
* Avoid a potential race when a CPU online notification doesn't fire on
* the onlined CPU but the tick creeps in before the event notification is
* run.
*/
{
}
{
{
/*
* Start update transaction.
*/
{
return;
}
/*
* Update the data.
*/
/*
* Complete transaction.
*/
}
}
}
/**
* Timer callback function for the sync and invariant GIP modes.
*
* @param pTimer The timer.
* @param pvUser Opaque pointer to the device extension.
* @param iTick The timer tick.
*/
static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
{
{
/*
* The calculations in supdrvGipUpdate() is somewhat timing sensitive,
* missing timer ticks is not an option for GIP because the GIP users
* will end up incrementing the time in 1ns per time getter call until
* there is a complete timer update. So, if the delta has yet to be
* calculated, we just pretend it is zero for now (the GIP users
* probably won't have it for a wee while either and will do the same).
*
* We could maybe on some platforms try cross calling a CPU with a
* working delta here, but it's not worth the hassle since the
* likelyhood of this happening is really low. On Windows, Linux, and
* Solaris timers fire on the CPU they were registered/started on.
* Darwin timers doesn't necessarily (they are high priority threads).
*/
Assert(!ASMIntAreEnabled());
{
}
}
}
/**
* Timer callback function for async GIP mode.
* @param pTimer The timer.
* @param pvUser Opaque pointer to the device extension.
* @param iTick The timer tick.
*/
{
/** @todo reset the transaction number and whatnot when iTick == 1. */
else
}
/*
*
*
* TSC Delta Measurements And Related Code
* TSC Delta Measurements And Related Code
* TSC Delta Measurements And Related Code
*
*
*/
/*
* Select TSC delta measurement algorithm.
*/
#if 0
# define GIP_TSC_DELTA_METHOD_1
#else
# define GIP_TSC_DELTA_METHOD_2
#endif
/** For padding variables to keep them away from other cache lines. Better too
* large than too small!
* @remarks Current AMD64 and x86 CPUs seems to use 64 bytes. There are claims
* that NetBurst had 128 byte cache lines while the 486 thru Pentium
* III had 32 bytes cache lines. */
#define GIP_TSC_DELTA_CACHE_LINE_SIZE 128
/**
* TSC delta measurment algorithm \#2 result entry.
*/
typedef struct SUPDRVTSCDELTAMETHOD2ENTRY
{
/**
* TSC delta measurment algorithm \#2 Data.
*/
typedef struct SUPDRVTSCDELTAMETHOD2
{
/** Padding to make sure the iCurSeqNo is in its own cache line. */
/** The current sequence number of this worker. */
/** Padding to make sure the iCurSeqNo is in its own cache line. */
/** Result table. */
/** Pointer to the data for TSC delta mesurment algorithm \#2 .*/
typedef SUPDRVTSCDELTAMETHOD2 *PSUPDRVTSCDELTAMETHOD2;
/**
* The TSC delta synchronization struct, version 2.
*
* The syncrhonization variable is completely isolated in its own cache line
* (provided our max cache line size estimate is correct).
*/
typedef struct SUPTSCDELTASYNC2
{
/** Padding to make sure the uVar1 is in its own cache line. */
/** The synchronization variable, holds values GIP_TSC_DELTA_SYNC_*. */
/** Sequence synchronizing variable used for post 'GO' synchronization. */
/** Padding to make sure the uVar1 is in its own cache line. */
/** Start RDTSC value. Put here mainly to save stack space. */
/** Copy of SUPDRVGIPTSCDELTARGS::cMaxTscTicks. */
typedef SUPTSCDELTASYNC2 *PSUPTSCDELTASYNC2;
/** Prestart wait. */
/** Prestart aborted. */
/** Ready (on your mark). */
/** Steady (get set). */
/** Go! */
/** Used by the verfication test. */
/** We reached the time limit. */
/** The other party won't touch the sync struct ever again. */
/**
* callback worker.
* @todo add
*/
typedef struct SUPDRVGIPTSCDELTARGS
{
/** The device extension. */
/** Pointer to the GIP CPU array entry for the worker. */
/** Pointer to the GIP CPU array entry for the master. */
/** The maximum number of ticks to spend in supdrvMeasureTscDeltaCallback.
* (This is what we need a rough TSC frequency for.) */
/** Used to abort synchronization setup. */
bool volatile fAbortSetup;
/** Padding to make sure the master variables live in its own cache lines. */
/** @name Master
* @{ */
/** The time the master spent in the MP worker. */
/** The iTry value when stopped at. */
/** Set if the run timed out. */
bool volatile fTimedOut;
/** Pointer to the master's synchronization struct (on stack). */
PSUPTSCDELTASYNC2 volatile pSyncMaster;
/** Master data union. */
union
{
/** Data (master) for delta verification. */
struct
{
/** Verification test TSC values for the master. */
} Verify;
/** Data (master) for measurement method \#2. */
struct
{
/** Data and sequence number. */
/** The lag setting for the next run. */
bool fLag;
/** Number of hits. */
} M2;
} uMaster;
/** The verifier verdict, VINF_SUCCESS if ok, VERR_OUT_OF_RANGE if not,
* VERR_TRY_AGAIN on timeout. */
#ifdef TSCDELTA_VERIFY_WITH_STATS
/** The maximum difference between TSC read during delta verification. */
/** The minimum difference between two TSC reads during verification. */
/** The bad TSC diff, worker relative to master (= worker - master).
* Negative value means the worker is behind the master. */
#endif
/** @} */
/** Padding to make sure the uVar1 is in its own cache line. */
/** @name Proletarian
* @{ */
/** Pointer to the worker's synchronization struct (on stack). */
PSUPTSCDELTASYNC2 volatile pSyncWorker;
/** The time the worker spent in the MP worker. */
/** Worker data union. */
union
{
/** Data (worker) for delta verification. */
struct
{
/** Verification test TSC values for the worker. */
} Verify;
/** Data (worker) for measurement method \#2. */
struct
{
/** Data and sequence number. */
/** The lag setting for the next run (set by master). */
bool fLag;
} M2;
} uWorker;
/** @} */
/** Padding to make sure the above is in its own cache line. */
typedef SUPDRVGIPTSCDELTARGS *PSUPDRVGIPTSCDELTARGS;
/** @name Macros that implements the basic synchronization steps common to
* the algorithms.
*
* Must be used from loop as the timeouts are implemented via 'break' statements
* at the moment.
*
* @{
*/
#if defined(DEBUG_bird) /* || defined(VBOX_STRICT) */
# define TSCDELTA_DBG_START_LOOP() do { iDbgCounter = 0; } while (0)
# define TSCDELTA_DBG_CHECK_LOOP() \
#else
# define TSCDELTA_DBG_VARS() ((void)0)
# define TSCDELTA_DBG_START_LOOP() ((void)0)
# define TSCDELTA_DBG_CHECK_LOOP() ((void)0)
#endif
#if 0
#else
# define TSCDELTA_DBG_SYNC_MSG(a_Args) ((void)0)
#endif
#if 0
#else
# define TSCDELTA_DBG_SYNC_MSG2(a_Args) ((void)0)
#endif
#if 0
#else
# define TSCDELTA_DBG_SYNC_MSG9(a_Args) ((void)0)
#endif
{
uint32_t iSync2Loops = 0;
/*
* The master tells the worker to get on it's mark.
*/
if (fIsMaster)
{
if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
{ /* likely*/ }
else
{
TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #1 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
return false;
}
}
/*
* Wait for the on your mark signal (ack in the master case). We process timeouts here.
*/
for (;;)
{
if (u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY)
break;
ASMNopPause();
/* Abort? */
if (u32Tmp != GIP_TSC_DELTA_SYNC2_READY)
{
TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #2 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
return false;
}
/* Check for timeouts every so often (not every loop in case RDTSC is
trapping or something). Must check the first time around. */
#if 0 /* For debugging the timeout paths. */
#endif
if ( ( (iSync2Loops & 0x3ff) == 0
#if 0 /* This is crazy, I know, but enable this code and the results are markedly better when enabled on the 1.4GHz AMD (debug). */
#endif
)
{
/* Try switch our own state into timeout mode so the master cannot tell us to 'GO',
ignore the timeout if we've got the go ahead already (simpler). */
if (ASMAtomicCmpXchgU32(&pMySync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_READY))
{
ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_STEADY);
return false;
}
}
iSync2Loops++;
}
/*
* Interrupts are now disabled and will remain disabled until we do
* TSCDELTA_MASTER_SYNC_AFTER / TSCDELTA_OTHER_SYNC_AFTER.
*/
/*
* The worker tells the master that it is on its mark and that the master
* need to get into position as well.
*/
if (!fIsMaster)
{
if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
{ /* likely */ }
else
{
TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #3 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
return false;
}
}
/*
* The master sends the 'go' to the worker and wait for ACK.
*/
if (fIsMaster)
{
if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
{ /* likely */ }
else
{
TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #4 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
return false;
}
}
/*
* Wait for the 'go' signal (ack in the master case).
*/
for (;;)
{
if (u32Tmp == GIP_TSC_DELTA_SYNC2_GO)
break;
{ /* likely */ }
else
{
TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #5 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
return false;
}
ASMNopPause();
}
/*
* The worker acks the 'go' (shouldn't fail).
*/
if (!fIsMaster)
{
if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
{ /* likely */ }
else
{
TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #6 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
return false;
}
}
/*
* Try enter mostly lockstep execution with it.
*/
for (;;)
{
ASMNopPause();
ASMNopPause();
if (iOtherSeq1 == iOtherSeq2)
return true;
/* Did the other guy give up? Should we give up? */
if ( iOtherSeq1 == UINT32_MAX
|| iOtherSeq2 == UINT32_MAX)
return true;
{
return true;
}
ASMNopPause();
}
}
if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, true /*fIsMaster*/, a_pfEFlags, a_pArgs))) \
{ /*likely*/ } \
else if (true) \
{ \
TSCDELTA_DBG_SYNC_MSG9(("sync/before/master: #89\n")); \
break; \
} else do {} while (0)
if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, false /*fIsMaster*/, a_pfEFlags, a_pArgs))) \
{ /*likely*/ } \
else if (true) \
{ \
TSCDELTA_DBG_SYNC_MSG9(("sync/before/other: #89\n")); \
break; \
} else do {} while (0)
{
/*
* Wait for the 'ready' signal. In the master's case, this means the
* worker has completed its data collection, while in the worker's case it
* means the master is done processing the data and it's time for the next
* loop iteration (or whatever).
*/
for (;;)
{
if ( u32Tmp == GIP_TSC_DELTA_SYNC2_READY
return true;
ASMNopPause();
{ /* likely */}
else
{
return false; /* shouldn't ever happen! */
}
ASMNopPause();
}
}
{ /* likely */ } \
else if (true) \
{ \
TSCDELTA_DBG_SYNC_MSG9(("sync/after/master: #97\n")); \
break; \
} else do {} while (0)
/* \
* Tell the worker that we're done processing the data and ready for the next round. \
*/ \
if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
{ /* likely */ } \
else if (true)\
{ \
break; \
} else do {} while (0)
if (true) { \
/* \
* Tell the master that we're done collecting data and wait for the next round to start. \
*/ \
if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
{ /* likely */ } \
else \
{ \
ASMSetFlags(a_fEFlags); \
break; \
} \
if (RT_LIKELY(supdrvTscDeltaSync2_After(a_pMySync, a_pOtherSync, false /*fIsMaster*/, a_fEFlags))) \
{ /* likely */ } \
else \
{ \
TSCDELTA_DBG_SYNC_MSG9(("sync/after/other: #98\n")); \
break; \
} \
} else do {} while (0)
/** @} */
#ifdef GIP_TSC_DELTA_METHOD_1
/**
* TSC delta measurment algorithm \#1 (GIP_TSC_DELTA_METHOD_1).
*
*
* We ignore the first few runs of the loop in order to prime the
* cache. Also, we need to be careful about using 'pause' instruction
* in critical busy-wait loops in this code - it can cause undesired
* behaviour with hyperthreading.
*
* We try to minimize the measurement error by computing the minimum
* read time of the compare statement in the worker by taking TSC
* measurements across it.
*
* It must be noted that the computed minimum read time is mostly to
* eliminate huge deltas when the worker is too early and doesn't by
* itself help produce more accurate deltas. We allow two times the
* computed minimum as an arbibtrary acceptable threshold. Therefore,
* it is still possible to get negative deltas where there are none
* when the worker is earlier. As long as these occasional negative
* deltas are lower than the time it takes to exit guest-context and
* the OS to reschedule EMT on a different CPU we won't expose a TSC
* that jumped backwards. It is because of the existence of the
* negative deltas we don't recompute the delta with the master and
* worker interchanged to eliminate the remaining measurement error.
*
*
* @param pMySync My synchronization structure.
* @param pOtherSync My partner's synchronization structure.
* @param fIsMaster Set if master, clear if worker.
* @param iTry The attempt number.
*/
static void supdrvTscDeltaMethod1Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
{
unsigned iLoop;
{
if (fIsMaster)
{
/*
* The master.
*/
("%#llx idMaster=%#x idWorker=%#x (idGipMaster=%#x)\n",
pGipCpuMaster->u64TSCSample, pGipCpuMaster->idCpu, pGipCpuWorker->idCpu, pArgs->pDevExt->idGipMaster));
do
{
/* Process the data. */
{
{
}
}
/* Reset our TSC sample and tell the worker to move on. */
}
else
{
/*
* The worker.
*/
/*
* Keep reading the TSC until we notice that the master has read his. Reading
* the TSC -after- the master has updated the memory is way too late. We thus
* compensate by trying to measure how long it took for the worker to notice
* the memory flushed from the master.
*/
do
{
uTscWorker = ASMReadTSC();
{
/* This is totally arbitrary a.k.a I don't like it but I have no better ideas for now. */
{
if (uCmpReadTime < uMinCmpReadTime)
}
else
}
else if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS)
{
if (uCmpReadTime < uMinCmpReadTime)
}
}
}
TSCDELTA_DBG_SYNC_MSG9(("sync/method1loop/%s: #92 iLoop=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iLoop,
/*
* We must reset the worker TSC sample value in case it gets picked as a
* GIP master later on (it's trashed above, naturally).
*/
if (!fIsMaster)
}
#endif /* GIP_TSC_DELTA_METHOD_1 */
#ifdef GIP_TSC_DELTA_METHOD_2
/*
* TSC delta measurement algorithm \#2 configuration and code - Experimental!!
*/
# define GIP_TSC_DELTA_M2_PRIMER_LOOPS 0
{
/*
* Look for matching entries in the master and worker tables.
*/
{
if (idxOther & 1)
{
idxOther >>= 1;
{
if (pArgs->uWorker.M2.Data.aResults[idxOther].iSeqOther == pArgs->uMaster.M2.Data.aResults[idxResult].iSeqMine)
{
? iDelta < iBestDelta
iBestDelta = iDelta;
cHits++;
}
}
}
}
/*
* Save the results.
*/
if (cHits > 2)
}
/**
* The core function of the 2nd TSC delta mesurment algorithm.
*
* The idea here is that we have the two CPUs execute the exact same code
* collecting a largish set of TSC samples. The code has one data dependency on
* the other CPU which intention it is to synchronize the execution as well as
* help cross references the two sets of TSC samples (the sequence numbers).
*
* The @a fLag parameter is used to modify the execution a tiny bit on one or
* both of the CPUs. When @a fLag differs between the CPUs, it is thought that
* it will help with making the CPUs enter lock step execution occationally.
*
*/
static void supdrvTscDeltaMethod2CollectData(PSUPDRVTSCDELTAMETHOD2 pMyData, uint32_t volatile *piOtherSeqNo, bool fLag)
{
while (cLeft-- > 0)
{
ASMSerializeInstruction(); /* Way better result than with ASMMemoryFenceSSE2() in this position! */
uTsc = ASMReadTSC();
pEntry++;
if (fLag)
ASMNopPause();
}
}
/**
* TSC delta measurment algorithm \#2 (GIP_TSC_DELTA_METHOD_2).
*
* See supdrvTscDeltaMethod2CollectData for algorithm details.
*
* @param pMySync My synchronization structure.
* @param pOtherSync My partner's synchronization structure.
* @param fIsMaster Set if master, clear if worker.
* @param iTry The attempt number.
*/
static void supdrvTscDeltaMethod2Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
{
unsigned iLoop;
{
if (fIsMaster)
{
/*
* Adjust the loop lag fudge.
*/
# if GIP_TSC_DELTA_M2_PRIMER_LOOPS > 0
{
/* Lag during the priming to be nice to everyone.. */
}
else
# endif
{
/* 25 % of the body without lagging. */
}
{
/* 25 % of the body with both lagging. */
}
else
{
/* 50% of the body with alternating lag. */
}
/*
* Sync up with the worker and collect data.
*/
supdrvTscDeltaMethod2CollectData(&pArgs->uMaster.M2.Data, &pArgs->uWorker.M2.Data.iCurSeqNo, pArgs->uMaster.M2.fLag);
/*
* Process the data.
*/
# if GIP_TSC_DELTA_M2_PRIMER_LOOPS > 0
if (iLoop >= GIP_TSC_DELTA_M2_PRIMER_LOOPS)
# endif
}
else
{
/*
* The worker.
*/
supdrvTscDeltaMethod2CollectData(&pArgs->uWorker.M2.Data, &pArgs->uMaster.M2.Data.iCurSeqNo, pArgs->uWorker.M2.fLag);
}
}
}
#endif /* GIP_TSC_DELTA_METHOD_2 */
{
/*PSUPGIPCPU pGipCpuWorker = pArgs->pWorker; - unused */
uint32_t i;
for (;;)
{
AssertCompile(RT_ELEMENTS(pArgs->uMaster.Verify.auTscs) == RT_ELEMENTS(pArgs->uWorker.Verify.auTscs));
if (fIsMaster)
{
/*
* Collect TSC, master goes first.
*/
{
/* Read, kick & wait #1. */
{
ASMNopPause();
}
/* Read, kick & wait #2. */
uTsc = ASMReadTSC();
{
ASMNopPause();
}
}
/*
* Process the data.
*/
#ifdef TSCDELTA_VERIFY_WITH_STATS
pArgs->iVerifyBadTscDiff = 0;
#endif
uTscWorker = 0;
{
/* Master vs previous worker entry. */
if (i > 0)
{
#ifdef TSCDELTA_VERIFY_WITH_STATS
#endif
if (iDiff < 0)
{
#ifdef TSCDELTA_VERIFY_WITH_STATS
#endif
break;
}
}
/* Worker vs master. */
#ifdef TSCDELTA_VERIFY_WITH_STATS
#endif
if (iDiff < 0)
{
#ifdef TSCDELTA_VERIFY_WITH_STATS
#endif
break;
}
}
/* Done. */
}
else
{
/*
* The worker, master leads.
*/
{
/* Wait, Read and Kick #1. */
{
ASMNopPause();
}
uTsc = ASMReadTSC();
/* Wait, Read and Kick #2. */
{
ASMNopPause();
}
uTsc = ASMReadTSC();
}
}
}
/*
* Timed out, please retry.
*/
return VERR_TIMEOUT;
}
/**
* Handles the special abort procedure during synchronization setup in
* supdrvMeasureTscDeltaCallbackUnwrapped().
*
* @returns 0 (dummy, ignored)
* @param pMySync Pointer to my sync structure.
* @param fIsMaster Set if we're the master, clear if worker.
* @param fTimeout Set if it's a timeout.
*/
DECL_NO_INLINE(static, int)
supdrvMeasureTscDeltaCallbackAbortSyncSetup(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, bool fIsMaster, bool fTimeout)
{
/*
* Clear our sync pointer and make sure the abort flag is set.
*/
if (fTimeout)
/*
* Make sure the other party is out of there and won't be touching our
* sync state again (would cause stack corruption).
*/
{
ASMNopPause();
ASMNopPause();
ASMNopPause();
}
return 0;
}
/**
* This is used by supdrvMeasureInitialTscDeltas() to read the TSC on two CPUs
* and compute the delta between them.
*
* To reduce code size a good when timeout handling was added, a dummy return
* value had to be added (saves 1-3 lines per timeout case), thus this
* 'Unwrapped' function and the dummy 0 return value.
*
* @returns 0 (dummy, ignored)
* @param idCpu The CPU we are current scheduled on.
* @param pArgs Pointer to a parameter package.
*
* @remarks Measuring TSC deltas between the CPUs is tricky because we need to
* read the TSC at exactly the same time on both the master and the
* worker CPUs. Due to DMA, bus arbitration, cache locality,
* contention, SMI, pipelining etc. there is no guaranteed way of
* doing this on x86 CPUs.
*/
{
int rc;
/* A bit of paranoia first. */
if (!pGipCpuMaster || !pGipCpuWorker)
return 0;
/*
* If the CPU isn't part of the measurement, return immediately.
*/
if ( !fIsMaster
return 0;
/*
* Set up my synchronization stuff and wait for the other party to show up.
*
* We don't wait forever since the other party may be off fishing (offline,
* spinning with ints disables, whatever), we must play nice to the rest of
* the system as this context generally isn't one in which we will get
* preempted and we may hold up a number of lower priority interrupts.
*/
/* Look for the partner, might not be here yet... Special abort considerations. */
iTry = 0;
{
ASMNopPause();
if ( (iTry++ & 0xff) == 0
ASMNopPause();
}
/* I found my partner, waiting to be found... Special abort considerations. */
if (fIsMaster)
if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* parnaoia */
iTry = 0;
{
ASMNopPause();
if ( (iTry++ & 0xff) == 0
{
if ( fIsMaster
&& !ASMAtomicCmpXchgU32(&MySync.uSyncVar, GIP_TSC_DELTA_SYNC2_PRESTART_ABORT, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT))
break; /* race #1: slave has moved on, handle timeout in loop instead. */
}
}
if (!fIsMaster)
if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* race #1 */
/** @todo Add a resumable state to pArgs so we don't waste time if we time
* out or something. Timeouts are legit, any of the two CPUs may get
* interrupted. */
/*
* Start by seeing if we have a zero delta between the two CPUs.
* This should normally be the case.
*/
rc = supdrvTscDeltaVerify(pArgs, &MySync, pOtherSync, fIsMaster, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
if (RT_SUCCESS(rc))
{
if (fIsMaster)
{
}
}
/*
* If the verification didn't time out, do regular delta measurements.
* We retry this until we get a reasonable value.
*/
else if (rc != VERR_TIMEOUT)
{
{
/*
* Check the state before we start.
*/
if ( u32Tmp != GIP_TSC_DELTA_SYNC2_READY
&& (fIsMaster || u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
{
TSCDELTA_DBG_SYNC_MSG(("sync/loop/%s: #0 iTry=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iTry, u32Tmp));
break;
}
/*
* Do the measurements.
*/
#ifdef GIP_TSC_DELTA_METHOD_1
#elif defined(GIP_TSC_DELTA_METHOD_2)
#else
# error "huh??"
#endif
/*
* Check the state.
*/
if ( u32Tmp != GIP_TSC_DELTA_SYNC2_READY
&& (fIsMaster || u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
{
if (fIsMaster)
else
break;
}
/*
* Success? If so, stop trying. Master decides.
*/
if (fIsMaster)
{
{
break;
}
}
}
if (fIsMaster)
}
/*
* End the synchroniziation dance. We tell the other that we're done,
* then wait for the same kind of reply.
*/
iTry = 0;
{
iTry++;
if ( iTry == 0
break; /* this really shouldn't happen. */
ASMNopPause();
}
/*
* Collect some runtime stats.
*/
if (fIsMaster)
else
return 0;
}
/**
* Callback used by supdrvMeasureInitialTscDeltas() to read the TSC on two CPUs
* and compute the delta between them.
*
* @param idCpu The CPU we are current scheduled on.
* @param pvUser1 Pointer to a parameter package (SUPDRVGIPTSCDELTARGS).
* @param pvUser2 Unused.
*/
static DECLCALLBACK(void) supdrvMeasureTscDeltaCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
{
}
/**
* Measures the TSC delta between the master GIP CPU and one specified worker
* CPU.
*
* @returns VBox status code.
* @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED on pure measurement
* failure.
* @param pDevExt Pointer to the device instance data.
* @param idxWorker The index of the worker CPU from the GIP's array of
* CPUs.
*
* @remarks This must be called with preemption enabled!
*/
{
int rc;
int rc2;
/* Validate input a bit. */
/*
* Don't attempt measuring the delta for the GIP master.
*/
{
return VINF_SUCCESS;
}
/*
* One measurement at at time, at least for now. We might be using
* broadcast IPIs so, so be nice to the rest of the system.
*/
#ifdef SUPDRV_USE_MUTEX_FOR_GIP
#else
#endif
if (RT_FAILURE(rc))
return rc;
/*
* If the CPU has hyper-threading and the APIC IDs of the master and worker are adjacent,
* try pick a different master. (This fudge only works with multi core systems.)
* ASSUMES related threads have adjacent APIC IDs. ASSUMES two threads per core.
*
* We skip this on AMDs for now as their HTT is different from intel's and
* it doesn't seem to have any favorable effect on the results.
*
* If the master is offline, we need a new master too, so share the code.
*/
&& ASMHasCpuId()
&& ASMIsValidStdRange(ASMCpuId_EAX(0))
&& !ASMIsAmdCpu()
|| !RTMpIsCpuOnline(idMaster) )
{
uint32_t i;
if ( i != iGipCpuMaster
&& i != idxWorker
{
iGipCpuMaster = i;
break;
}
}
{
/*
* Initialize data package for the RTMpOnPair callback.
*/
if (pArgs)
{
/*
* Do the RTMpOnPair call. We reset i64TSCDelta first so we
* and supdrvMeasureTscDeltaCallback can use it as a success check.
*/
/** @todo Store the i64TSCDelta result in pArgs first? Perhaps deals with
* that when doing the restart loop reorg. */
if (RT_SUCCESS(rc))
{
#if 0
#endif
#if 0
SUPR0Printf("rcVerify=%d iVerifyBadTscDiff=%lld cMinVerifyTscTicks=%lld cMaxVerifyTscTicks=%lld\n",
#endif
{
/*
* Work the TSC delta applicability rating. It starts
* optimistic in supdrvGipInit, we downgrade it here.
*/
else
{
}
}
else
}
/** @todo return try-again if we get an offline CPU error. */
}
else
rc = VERR_NO_MEMORY;
}
else
/*
* We're done now.
*/
#ifdef SUPDRV_USE_MUTEX_FOR_GIP
#else
#endif
return rc;
}
/**
* Clears TSC delta related variables.
*
* Clears all TSC samples as well as the delta synchronization variable on the
* all the per-CPU structs. Optionally also clears the per-cpu deltas too.
*
* @param pDevExt Pointer to the device instance data.
* @param fClearDeltas Whether the deltas are also to be cleared.
*/
{
unsigned iCpu;
{
if (fClearDeltas)
}
}
/**
* Performs the initial measurements of the TSC deltas between CPUs.
*
* This is called by supdrvGipCreate or triggered by it if threaded.
*
* @returns VBox status code.
* @param pDevExt Pointer to the device instance data.
*
* @remarks Must be called only after supdrvGipInitOnCpu() as this function uses
* idCpu, GIP's online CPU set which are populated in
* supdrvGipInitOnCpu().
*/
{
unsigned iCpu;
unsigned iOddEven;
int rc = VINF_SUCCESS;
/*
* Pick the first CPU online as the master TSC and make it the new GIP master based
* on the APIC ID.
*
* Technically we can simply use "idGipMaster" but doing this gives us master as CPU 0
*/
{
if (idxCpu != UINT16_MAX)
{
{
break;
}
}
}
/*
* If there is only a single CPU online we have nothing to do.
*/
{
return VINF_SUCCESS;
}
/*
* Loop thru the GIP CPU array and get deltas for each CPU (except the
* master). We do the CPUs with the even numbered APIC IDs first so that
* we've got alternative master CPUs to pick from on hyper-threaded systems.
*/
{
{
{
if (RT_FAILURE(rc))
{
SUPR0Printf("supdrvMeasureTscDeltaOne failed. rc=%d CPU[%u].idCpu=%u Master[%u].idCpu=%u\n", rc, iCpu,
break;
}
{
SUPR0Printf("One or more CPUs transitioned between online & offline states. I'm confused, retry...\n");
rc = VERR_TRY_AGAIN;
break;
}
}
}
}
return rc;
}
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
/**
* Switches the TSC-delta measurement thread into the butchered state.
*
* @returns VBox status code.
* @param pDevExt Pointer to the device instance data.
* @param fSpinlockHeld Whether the TSC-delta spinlock is held or not.
* @param pszFailed An error message to log.
* @param rcFailed The error code to exit the thread with.
*/
static int supdrvTscDeltaThreadButchered(PSUPDRVDEVEXT pDevExt, bool fSpinlockHeld, const char *pszFailed, int rcFailed)
{
if (!fSpinlockHeld)
return rcFailed;
}
/**
* The TSC-delta measurement thread.
*
* @returns VBox status code.
* @param hThread The thread handle.
* @param pvUser Opaque pointer to the device instance data.
*/
{
bool fInitialMeasurement = true;
int rc = VERR_INTERNAL_ERROR_2;
for (;;)
{
/*
* Switch on the current state.
*/
switch (enmState)
{
{
if (RT_FAILURE(rc))
/* fall thru */
}
{
/* Simple adaptive timeout. */
if (cConsecutiveTimeouts++ == 10)
{
cConsecutiveTimeouts = 0;
}
if ( RT_FAILURE(rc)
&& rc != VERR_TIMEOUT)
break;
}
{
rc = RTSemEventSignal(pDevExt->hTscDeltaEvent); /* (Safe on windows as long as spinlock isn't IRQ safe.) */
if (RT_FAILURE(rc))
RTThreadSleep(1);
/* fall thru */
}
{
cConsecutiveTimeouts = 0;
if (fInitialMeasurement)
{
int cTries = 8;
int cMsWaitPerTry = 10;
fInitialMeasurement = false;
do
{
if ( RT_SUCCESS(rc)
|| ( RT_FAILURE(rc)
&& rc != VERR_TRY_AGAIN
&& rc != VERR_CPU_OFFLINE))
{
break;
}
} while (cTries-- > 0);
}
else
{
unsigned iCpu;
/* Measure TSC-deltas only for the CPUs that are in the set. */
rc = VINF_SUCCESS;
{
{
{
}
else
{
/*
* mark the delta as fine to get the timer thread off our back.
*/
}
}
}
}
break;
}
return VINF_SUCCESS;
default:
return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "Invalid state", VERR_INVALID_STATE);
}
}
return rc;
}
/**
* Waits for the TSC-delta measurement thread to respond to a state change.
*
* @returns VINF_SUCCESS on success, VERR_TIMEOUT if it doesn't respond in time,
* other error code on internal error.
*
* @param pThis Pointer to the grant service instance data.
* @param enmCurState The current state.
* @param enmNewState The new state we're waiting for it to enter.
*/
{
/*
* Wait a short while for the expected state transition.
*/
int rc;
{
rc = VINF_SUCCESS;
}
{
/*
* Wait longer if the state has not yet transitioned to the one we want.
*/
if ( RT_SUCCESS(rc)
|| rc == VERR_TIMEOUT)
{
/*
* Check the state whether we've succeeded.
*/
if (enmState == enmNewState)
rc = VINF_SUCCESS;
else if (enmState == enmCurState)
{
rc = VERR_TIMEOUT;
OSDBGPRINT(("supdrvTscDeltaThreadWait: timed out state transition. enmState=%d enmNewState=%d\n", enmState,
enmNewState));
}
else
{
OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d, expected %d\n", enmCurState,
enmState, enmNewState));
}
}
else
}
else
{
OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d\n", enmCurState, enmNewState));
}
return rc;
}
/**
* Signals the TSC-delta thread to start measuring TSC-deltas.
*
* @param pDevExt Pointer to the device instance data.
*/
{
{
{
}
}
}
/**
* Terminates the actual thread running supdrvTscDeltaThread().
*
* This is an internal worker function for supdrvTscDeltaThreadInit() and
* supdrvTscDeltaTerm().
*
* @param pDevExt Pointer to the device instance data.
*/
{
int rc;
if (RT_FAILURE(rc))
{
/* Signal a few more times before giving up. */
int cTriesLeft = 5;
while (--cTriesLeft > 0)
{
if (rc != VERR_TIMEOUT)
break;
}
}
}
/**
* Initializes and spawns the TSC-delta measurement thread.
*
* A thread is required for servicing re-measurement requests from events like
* under all contexts on all OSs.
*
* @returns VBox status code.
* @param pDevExt Pointer to the device instance data.
*
* @remarks Must only be called -after- initializing GIP and setting up MP
* notifications!
*/
{
int rc;
rc = RTSpinlockCreate(&pDevExt->hTscDeltaSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "VBoxTscSpnLck");
if (RT_SUCCESS(rc))
{
if (RT_SUCCESS(rc))
{
if (RT_SUCCESS(rc))
{
rc = supdrvTscDeltaThreadWait(pDevExt, kTscDeltaThreadState_Creating, kTscDeltaThreadState_Listening);
if (RT_SUCCESS(rc))
{
return rc;
}
}
else
}
else
}
else
return rc;
}
/**
* Terminates the TSC-delta measurement thread and cleanup.
*
* @param pDevExt Pointer to the device instance data.
*/
{
{
}
{
}
{
}
}
#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
/**
* Measure the TSC delta for the CPU given by its CPU set index.
*
* @returns VBox status code.
* @retval VERR_INTERRUPTED if interrupted while waiting.
* @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED if we were unable to get a
* measurment.
* @retval VERR_CPU_OFFLINE if the specified CPU is offline.
* @retval VERR_CPU_OFFLINE if the specified CPU is offline.
*
* @param pSession The caller's session. GIP must've been mapped.
* @param iCpuSet The CPU set index of the CPU to measure.
* @param fFlags Flags, SUP_TSCDELTA_MEASURE_F_XXX.
* @param cMsWaitRetry Number of milliseconds to wait between each retry.
* @param cMsWaitThread Number of milliseconds to wait for the thread to get
* ready.
* @param cTries Number of times to try, pass 0 for the default.
*/
SUPR0DECL(int) SUPR0TscDeltaMeasureBySetIndex(PSUPDRVSESSION pSession, uint32_t iCpuSet, uint32_t fFlags,
{
int rc;
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
#endif
/*
* Validate and adjust the input.
*/
if (!pSession->fGipReferenced)
return VERR_WRONG_ORDER;
return VERR_INVALID_FLAGS;
/*
* The request is a noop if the TSC delta isn't being used.
*/
return VINF_SUCCESS;
if (cTries == 0)
cTries = 12;
else if (cTries > 256)
cTries = 256;
if (cMsWaitRetry == 0)
cMsWaitRetry = 2;
else if (cMsWaitRetry > 1000)
cMsWaitRetry = 1000;
#ifdef SUPDRV_USE_TSC_DELTA_THREAD
/*
* Has the TSC already been measured and we're not forced to redo it?
*/
&& !(fFlags & SUP_TSCDELTA_MEASURE_F_FORCE))
return VINF_SUCCESS;
/*
* Asynchronous request? Forward it to the thread, no waiting.
*/
{
/** @todo Async. doesn't implement options like retries, waiting. We'll need
* to pass those options to the thread somehow and implement it in the
{
rc = VINF_SUCCESS;
}
return VINF_SUCCESS;
}
/*
* If a TSC-delta measurement request is already being serviced by the thread,
* wait 'cTries' times if a retry-timeout is provided, otherwise bail as busy.
*/
{
{ /* Must wait, the thread is busy. */ }
else if (enmState == kTscDeltaThreadState_WaitAndMeasure)
{ /* Must wait, this state only says what will happen next. */ }
else if (enmState == kTscDeltaThreadState_Terminating)
{ /* Must wait, this state only says what should happen next. */ }
else
break; /* All other states, the thread is either idly listening or dead. */
/* Wait or fail. */
if (cMsWaitThread == 0)
if (cMsElapsed >= cMsWaitThread)
if (rc == VERR_INTERRUPTED)
return rc;
}
#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
/*
* Try measure the TSC delta the given number of times.
*/
for (;;)
{
/* Unless we're forced to measure the delta, check whether it's done already. */
if ( !(fFlags & SUP_TSCDELTA_MEASURE_F_FORCE)
{
rc = VINF_SUCCESS;
break;
}
/* Measure it. */
{
break;
}
/* Retry? */
if (cTries <= 1)
break;
cTries--;
/* Always delay between retries (be nice to the rest of the system
and avoid the BSOD hounds). */
if (rc == VERR_INTERRUPTED)
break;
}
return rc;
}
/**
* Service a TSC-delta measurement request.
*
* @returns VBox status code.
* @param pDevExt Pointer to the device instance data.
* @param pSession The support driver session.
* @param pReq Pointer to the TSC-delta measurement request.
*/
int VBOXCALL supdrvIOCtl_TscDeltaMeasure(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCDELTAMEASURE pReq)
{
/*
*/
return VERR_INVALID_CPU_ID;
if (iCpuSet >= RTCPUSET_MAX_CPUS)
return VERR_INVALID_CPU_ID;
fFlags = 0;
cTries);
}
/**
* Reads TSC with delta applied.
*
* Will try to resolve delta value INT64_MAX before applying it. This is the
* main purpose of this function, to handle the case where the delta needs to be
* determined.
*
* @returns VBox status code.
* @param pDevExt Pointer to the device instance data.
* @param pSession The support driver session.
* @param pReq Pointer to the TSC-read request.
*/
{
int rc;
/*
* Validate. We require the client to have mapped GIP (no asserting on
* ring-3 preconditions).
*/
return VERR_WRONG_ORDER;
/*
* We're usually here because we need to apply delta, but we shouldn't be
* upset if the GIP is some different mode.
*/
{
for (;;)
{
/*
* Start by gathering the data, using CLI for disabling preemption
* while we do that.
*/
int iGipCpu;
{
/*
* If we're lucky we've got a delta, but no predicitions here
* as this I/O control is normally only used when the TSC delta
* is set to INT64_MAX.
*/
{
rc = VINF_SUCCESS;
break;
}
/* Give up after a few times. */
if (cTries >= 4)
{
break;
}
/* Need to measure the delta an try again. */
/** @todo should probably delay on failure... dpc watchdogs */
}
else
{
/* This really shouldn't happen. */
break;
}
}
}
else
{
/*
* No delta to apply. Easy. Deal with preemption the lazy way.
*/
int iGipCpu;
else
rc = VINF_SUCCESS;
}
return rc;
}