9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * VBoxDrv - The VirtualBox Support Driver - Common code for GIP.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Copyright (C) 2006-2015 Oracle Corporation
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * This file is part of VirtualBox Open Source Edition (OSE), as
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * available from http://www.virtualbox.org. This file is free software;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * you can redistribute it and/or modify it under the terms of the GNU
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * General Public License (GPL) as published by the Free Software
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Foundation, in version 2 as it comes in the "COPYING" file of the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * The contents of this file may alternatively be used under the terms
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * of the Common Development and Distribution License Version 1.0
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * VirtualBox OSE distribution, in which case the provisions of the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * CDDL are applicable instead of those of the GPL.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * You may elect to license modified versions of this file under the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * terms and conditions of either the GPL or the CDDL or both.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/*******************************************************************************
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync* Header Files *
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync*******************************************************************************/
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync#if defined(RT_OS_DARWIN) || defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/*******************************************************************************
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync* Defined Constants And Macros *
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync*******************************************************************************/
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** The frequency by which we recalculate the u32UpdateHz and
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * u32UpdateIntervalNS GIP members. The value must be a power of 2.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Warning: Bumping this too high might overflow u32UpdateIntervalNS.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** A reserved TSC value used for synchronization as well as measurement of
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * TSC deltas. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** The number of TSC delta measurement loops in total (includes primer and
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * read-time loops). */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** The number of cache primer loops. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** The number of loops until we keep computing the minumum read time. */
5acdcd39dfd0adec57d6e85838e999ba952ee49dvboxsync/** The TSC frequency refinement period in seconds.
5acdcd39dfd0adec57d6e85838e999ba952ee49dvboxsync * The timer fires after 200ms, then every second, this value just says when
5acdcd39dfd0adec57d6e85838e999ba952ee49dvboxsync * to stop it after that. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_PRACTICALLY_ZERO rating */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_ROUGHLY_ZERO rating */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** The TSC delta value for the initial GIP master - 0 in regular builds.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * To test the delta code this can be set to a non-zero value. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(170139095182512) /* 0x00009abd9854acb0 */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(0)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncAssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS < GIP_TSC_DELTA_READ_TIME_LOOPS);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncAssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS < GIP_TSC_DELTA_LOOPS);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** @def VBOX_SVN_REV
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * The makefile should define this if it can. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync#if 0 /* Don't start the GIP timers. Useful when debugging the IPRT timer code. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/*******************************************************************************
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync* Internal Functions *
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync*******************************************************************************/
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsyncstatic void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsyncstatic void supdrvTscResetSamples(PSUPDRVDEVEXT pDevExt, bool fClearDeltas);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsyncstatic void supdrvTscDeltaThreadStartMeasurement(PSUPDRVDEVEXT pDevExt, bool fForceAll);
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsyncstatic int supdrvMeasureInitialTscDeltas(PSUPDRVDEVEXT pDevExt);
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsyncstatic int supdrvMeasureTscDeltaOne(PSUPDRVDEVEXT pDevExt, uint32_t idxWorker);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/*******************************************************************************
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync* Global Variables *
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync*******************************************************************************/
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncDECLEXPORT(PSUPGLOBALINFOPAGE) g_pSUPGlobalInfoPage = NULL;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Misc Common GIP Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Misc Common GIP Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Misc Common GIP Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Finds the GIP CPU index corresponding to @a idCpu.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns GIP CPU array index, UINT32_MAX if not found.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pGip The GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The CPU ID.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic uint32_t supdrvGipFindCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Mapping and Unmapping Related Code.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Mapping and Unmapping Related Code.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Mapping and Unmapping Related Code.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * (Re-)initializes the per-cpu structure prior to starting or resuming the GIP
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * updating.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pGip Pointer to the GIP.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pGipCpu The per CPU structure for this CPU.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param u64NanoTS The current time.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic void supdrvGipReInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Here we don't really care about applying the TSC delta. The re-initialization of this
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * value is not relevant especially while (re)starting the GIP as the first few ones will
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * be ignored anyway, see supdrvGipDoUpdateCpu().
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pGipCpu->u64TSC = ASMReadTSC() - pGipCpu->u32UpdateIntervalTSC;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Set the current TSC and NanoTS value for the CPU.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param idCpu The CPU ID. Unused - we have to use the APIC ID.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pvUser1 Pointer to the ring-0 GIP mapping.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pvUser2 Pointer to the variable holding the current time.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic DECLCALLBACK(void) supdrvGipReInitCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser1;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync unsigned iCpu = pGip->aiCpuFromApicId[ASMGetApicId()];
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (RT_LIKELY(iCpu < pGip->cCpus && pGip->aCPUs[iCpu].idCpu == idCpu))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipReInitCpu(pGip, &pGip->aCPUs[iCpu], *(uint64_t *)pvUser2);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * State structure for supdrvGipDetectGetGipCpuCallback.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /** Bitmap of APIC IDs that has been seen (initialized to zero).
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Used to detect duplicate APIC IDs (paranoia). */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /** Mask of supported GIP CPU getter methods (SUPGIPGETCPU_XXX) (all bits set
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * initially). The callback clears the methods not detected. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /** The first callback detecting any kind of range issues (initialized to
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * NIL_RTCPUID). */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** Pointer to state structure for supdrvGipDetectGetGipCpuCallback. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsynctypedef SUPDRVGIPDETECTGETCPU *PSUPDRVGIPDETECTGETCPU;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Checks for alternative ways of getting the CPU ID.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * This also checks the APIC ID, CPU ID and CPU set index values against the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * GIP tables.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param idCpu The CPU ID. Unused - we have to use the APIC ID.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pvUser1 Pointer to the state structure.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pvUser2 Pointer to the GIP.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic DECLCALLBACK(void) supdrvGipDetectGetGipCpuCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync PSUPDRVGIPDETECTGETCPU pState = (PSUPDRVGIPDETECTGETCPU)pvUser1;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser2;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertMsg(idCpu == RTMpCpuId(), ("idCpu=%#x RTMpCpuId()=%#x\n", idCpu, RTMpCpuId())); /* paranoia^3 */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Check that the CPU ID and CPU set index are interchangable.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Check whether the IDTR.LIMIT contains a CPU number.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync uint16_t const cbIdt = sizeof(X86DESC64SYSTEM) * 256;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync fSupported |= SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Check whether RDTSCP is an option.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if ( ASMIsValidExtRange(ASMCpuId_EAX(UINT32_C(0x80000000)))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync && (ASMCpuId_EDX(UINT32_C(0x80000001)) & X86_CPUID_EXT_FEATURE_EDX_RDTSCP) )
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync fSupported |= SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Check that the APIC ID is unique.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (RT_LIKELY( idApic < RT_ELEMENTS(pGip->aiCpuFromApicId)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync && !ASMAtomicBitTestAndSet(pState->bmApicId, idApic)))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertCompile(sizeof(pState->bmApicId) * 8 == RT_ELEMENTS(pGip->aiCpuFromApicId));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - duplicate APIC ID.\n",
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Check that the iCpuSet is within the expected range.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync || (unsigned)iCpuSet >= RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU set index is out of range.\n",
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU id/index roundtrip problem: %#x\n",
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Update the supported feature mask before we return.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Increase the timer freqency on hosts where this is possible (NT).
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * The idea is that more interrupts is better for us... Also, it's better than
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * we increase the timer frequence, because we might end up getting inaccurate
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * callbacks if someone else does it.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pDevExt Sets u32SystemTimerGranularityGrant if increased.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic void supdrvGipRequestHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if ( RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 976563 /* 1024 HZ */, &u32SystemResolution))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync || RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1000000 /* 1000 HZ */, &u32SystemResolution))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync || RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1953125 /* 512 HZ */, &u32SystemResolution))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync || RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 2000000 /* 500 HZ */, &u32SystemResolution))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync Assert(RTTimerGetSystemGranularity() <= u32SystemResolution);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pDevExt->u32SystemTimerGranularityGrant = u32SystemResolution;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Undoes supdrvGipRequestHigherTimerFrequencyFromSystem.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pDevExt Clears u32SystemTimerGranularityGrant.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic void supdrvGipReleaseHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync int rc2 = RTTimerReleaseSystemGranularity(pDevExt->u32SystemTimerGranularityGrant);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Maps the GIP into userspace and/or get the physical address of the GIP.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns IPRT status code.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pSession Session to which the GIP mapping should belong.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param ppGipR3 Where to store the address of the ring-3 mapping. (optional)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pHCPhysGip Where to store the physical address. (optional)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @remark There is no reference counting on the mapping, so one call to this function
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * count globally as one reference. One call to SUPR0GipUnmap() is will unmap GIP
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * and remove the session as a GIP user.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncSUPR0DECL(int) SUPR0GipMap(PSUPDRVSESSION pSession, PRTR3PTR ppGipR3, PRTHCPHYS pHCPhysGip)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogFlow(("SUPR0GipMap: pSession=%p ppGipR3=%p pHCPhysGip=%p\n", pSession, ppGipR3, pHCPhysGip));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertPtrNullReturn(ppGipR3, VERR_INVALID_POINTER);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertPtrNullReturn(pHCPhysGip, VERR_INVALID_POINTER);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTR0MemObjMapUser(&pSession->GipMapObjR3, pDevExt->GipMemObj, (RTR3PTR)-1, 0,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pGipR3 = RTR0MemObjAddressR3(pSession->GipMapObjR3);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Get physical address.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Reference globally.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * GIP starts/resumes updating again. On windows we bump the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * host timer frequency to make sure we don't get stuck in guest
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * mode and to get better timer (and possibly clock) accuracy.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * document me
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (pGipR0->aCPUs[0].u32TransactionId != 2 /* not the first time */)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync unsigned i;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync ASMAtomicUoWriteU32(&pGipR0->aCPUs[i].u32TransactionId,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync (pGipR0->aCPUs[i].u32TransactionId + GIP_UPDATEHZ_RECALC_FREQ * 2)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync ASMAtomicWriteU64(&pGipR0->u64NanoTSLastUpdateHz, 0);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * document me
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync u64NanoTS = RTTimeSystemNanoTS() - pGipR0->u32UpdateIntervalNS;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipReInitCpu(pGipR0, &pGipR0->aCPUs[0], u64NanoTS);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync RTMpOnAll(supdrvGipReInitCpuCallback, pGipR0, &u64NanoTS);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Detect alternative ways to figure the CPU ID in ring-3 and
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * raw-mode context. Check the sanity of the APIC IDs, CPU IDs,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * and CPU set indexes while we're at it.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync RT_BZERO((void *)&DetectState.bmApicId, sizeof(DetectState.bmApicId));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTMpOnAll(supdrvGipDetectGetGipCpuCallback, &DetectState, pGipR0);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogRel(("SUPR0GipMap: fGetGipCpu=%#x\n", DetectState.fSupported));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogRel(("SUPR0GipMap: No supported ways of getting the APIC ID or CPU number in ring-3! (%#x)\n",
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogRel(("SUPR0GipMap: APIC ID, CPU ID or CPU set index problem detected on CPU #%u (%#x)!\n",
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync DetectState.idCpuProblem, DetectState.idCpuProblem));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Start the GIP timer if all is well..
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTTimerStart(pDevExt->pGipTimer, 0 /* fire ASAP */); AssertRC(rc);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Bail out on error.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync int rc2 = RTR0MemObjFree(pSession->GipMapObjR3, false); AssertRC(rc2);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Write returns.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync OSDBGPRINT(("SUPR0GipMap: returns %d *pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void *)pGipR3));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogFlow(( "SUPR0GipMap: returns %d *pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void *)pGipR3));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Unmaps any user mapping of the GIP and terminates all GIP access
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * from this session.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns IPRT status code.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pSession Session to which the GIP mapping should belong.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncSUPR0DECL(int) SUPR0GipUnmap(PSUPDRVSESSION pSession)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync OSDBGPRINT(("SUPR0GipUnmap: pSession=%p pGip=%p GipMapObjR3=%p\n",
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pSession->GipMapObjR3 != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pSession->GipMapObjR3) : NULL,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogFlow(("SUPR0GipUnmap: pSession=%p\n", pSession));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Unmap anything?
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Dereference global GIP.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync LogFlow(("SUPR0GipUnmap: Suspends GIP updating\n"));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTTimerStop(pDevExt->pGipTimer); AssertRC(rc); rc = VINF_SUCCESS;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Gets the GIP pointer.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns Pointer to the GIP or NULL.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Initialization, Termination and CPU Offline / Online Related Code.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Initialization, Termination and CPU Offline / Online Related Code.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Initialization, Termination and CPU Offline / Online Related Code.
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * Used by supdrvInitRefineInvariantTscFreqTimer and supdrvGipInitMeasureTscFreq
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * to update the TSC frequency related GIP variables.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * @param pGip The GIP.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * @param nsElapsed The number of nano seconds elapsed.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * @param cElapsedTscTicks The corresponding number of TSC ticks.
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync * @param iTick The tick number for debugging.
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsyncstatic void supdrvGipInitSetCpuFreq(PSUPGLOBALINFOPAGE pGip, uint64_t nsElapsed, uint64_t cElapsedTscTicks, uint32_t iTick)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Calculate the frequency.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uCpuHz = ASMMultU64ByU32DivByU32(cElapsedTscTicks, RT_NS_1SEC, (uint32_t)nsElapsed);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync RTUInt128MulU64ByU64(&Tmp, cElapsedTscTicks, RT_NS_1SEC_64);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync RTUInt128Div(&CpuHz, &Tmp, RTUInt128AssignU64(&Divisor, nsElapsed));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Update the GIP.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync ASMAtomicWriteU64(&pGip->aCPUs[0].u64CpuHz, uCpuHz);
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync /* For inspecting the frequency calcs using tstGIP-2, debugger or similar. */
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync ASMAtomicWriteU64(&pGip->aCPUs[iTick + 1].u64CpuHz, uCpuHz);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Timer callback function for TSC frequency refinement in invariant GIP mode.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * This is started during driver init and fires once
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * GIP_TSC_REFINE_PERIOD_IN_SECS seconds later.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pTimer The timer.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser Opaque pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param iTick The timer tick.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsyncstatic DECLCALLBACK(void) supdrvInitRefineInvariantTscFreqTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Paranoia. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync AssertReturnVoid(pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC);
e3ca1fe5aceb4641ff101c920fc09c74754e3e46vboxsync * If we got a power event, stop the refinement process.
d2bf529b7f81219efcdc932bee290467a2b04e24vboxsync * Read the TSC and time, noting which CPU we are on.
5acdcd39dfd0adec57d6e85838e999ba952ee49dvboxsync * Don't bother spinning until RTTimeSystemNanoTS changes, since on
5acdcd39dfd0adec57d6e85838e999ba952ee49dvboxsync * systems where it matters we're in a context where we cannot waste that
5acdcd39dfd0adec57d6e85838e999ba952ee49dvboxsync * much time (DPC watchdog, called from clock interrupt).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync cNsElapsed = nsNow - pDevExt->nsStartInvarTscRefine;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync cTscTicksElapsed = uTsc - pDevExt->uTscStartInvarTscRefine;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * If the above measurement was taken on a different CPU than the one we
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * started the process on, cTscTicksElapsed will need to be adjusted with
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * the TSC deltas of both the CPUs.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * We ASSUME that the delta calculation process takes less time than the
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * TSC frequency refinement timer. If it doesn't, we'll complain and
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * drop the frequency refinement.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Note! We cannot entirely trust enmUseTscDelta here because it's
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * downgraded after each delta calculation.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync && pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint32_t iStartCpuSet = RTMpCpuIdToSetIndex(pDevExt->idCpuInvarTscRefine);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint16_t iStartGipCpu = iStartCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync ? pGip->aiCpuFromCpuSetIdx[iStartCpuSet] : UINT16_MAX;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint16_t iStopGipCpu = iStopCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync ? pGip->aiCpuFromCpuSetIdx[iStopCpuSet] : UINT16_MAX;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
874970279c503ed890db044712b8e5eaa6f6f82avboxsync if (RT_LIKELY(iStartTscDelta != INT64_MAX && iStopTscDelta != INT64_MAX))
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /* cTscTicksElapsed = (uTsc - iStopTscDelta) - (pDevExt->uTscStartInvarTscRefine - iStartTscDelta); */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync cTscTicksElapsed += iStartTscDelta - iStopTscDelta;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Allow 5 times the refinement period to elapse before we give up on the TSC delta
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * calculations.
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync else if (cNsElapsed > GIP_TSC_REFINE_PERIOD_IN_SECS * 5 * RT_NS_1SEC_64)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync SUPR0Printf("vboxdrv: Failed to refine invariant TSC frequency because deltas are unavailable after %u (%u) seconds\n",
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync (uint32_t)(cNsElapsed / RT_NS_1SEC), GIP_TSC_REFINE_PERIOD_IN_SECS);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync iStartCpuSet, iStartGipCpu, iStartTscDelta, iStopCpuSet, iStopGipCpu, iStopTscDelta);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Calculate and update the CPU frequency variables in GIP.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * If there is a GIP user already and we've already refined the frequency
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * a couple of times, don't update it as we want a stable frequency value
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * for all VMs.
b0deb52d71d46987430324c2e39de46a2791a738vboxsync supdrvGipInitSetCpuFreq(pGip, cNsElapsed, cTscTicksElapsed, (uint32_t)iTick);
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync * Stop the timer once we've reached the defined refinement period.
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync if (cNsElapsed > GIP_TSC_REFINE_PERIOD_IN_SECS * RT_NS_1SEC_64)
e3ca1fe5aceb4641ff101c920fc09c74754e3e46vboxsync * @callback_method_impl{FNRTPOWERNOTIFICATION}
e3ca1fe5aceb4641ff101c920fc09c74754e3e46vboxsyncstatic DECLCALLBACK(void) supdrvGipPowerNotificationCallback(RTPOWEREVENT enmEvent, void *pvUser)
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * If the TSC frequency refinement timer is running, we need to cancel it so it
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * doesn't screw up the frequency after a long suspend.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * Recalculate all TSC-deltas on host resume as it may have changed, seen
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * on Windows 7 running on the Dell Optiplex Intel Core i5-3570.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync ASMAtomicWriteBool(&pDevExt->fInvTscRefinePowerEvent, true);
6f8f54585e02c203bef2e7ec2f59829163fdc971vboxsync && pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync RTCpuSetCopy(&pDevExt->TscDeltaCpuSet, &pGip->OnlineCpuSet);
e3ca1fe5aceb4641ff101c920fc09c74754e3e46vboxsync ASMAtomicWriteBool(&pDevExt->fInvTscRefinePowerEvent, true);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Start the TSC-frequency refinment timer for the invariant TSC GIP mode.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * We cannot use this in the synchronous and asynchronous tsc GIP modes because
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * the CPU may change the TSC frequence between now and when the timer fires
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * (supdrvInitAsyncRefineTscTimer).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @param pDevExt Pointer to the device instance data.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @param pGip Pointer to the GIP.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsyncstatic void supdrvGipInitStartTimerForRefiningInvariantTscFreq(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip)
e3ca1fe5aceb4641ff101c920fc09c74754e3e46vboxsync * Register a power management callback.
e3ca1fe5aceb4641ff101c920fc09c74754e3e46vboxsync rc = RTPowerNotificationRegister(supdrvGipPowerNotificationCallback, pDevExt);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Record the TSC and NanoTS as the starting anchor point for refinement
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * of the TSC. We try get as close to a clock tick as possible on systems
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * which does not provide high resolution time.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync pDevExt->nsStartInvarTscRefine = RTTimeSystemNanoTS();
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Create a timer that runs on the same CPU so we won't have a depencency
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * on the TSC-delta and can run in parallel to it. On systems that does not
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * implement CPU specific timers we'll apply deltas in the timer callback,
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * just like we do for CPUs going offline.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * The longer the refinement interval the better the accuracy, at least in
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * theory. If it's too long though, ring-3 may already be starting its
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * first VMs before we're done. On most systems we will be loading the
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * support driver during boot and VMs won't be started for a while yet,
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * it is really only a problem during development (especially with
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * on-demand driver starting on windows).
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * To avoid wasting time doing a long supdrvGipInitMeasureTscFreq() call
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * to calculate the frequency during driver loading, the timer is set
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * to fire after 200 ms the first time. It will then reschedule itself
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * to fire every second until GIP_TSC_REFINE_PERIOD_IN_SECS has been
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * reached or it notices that there is a user land client with GIP
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * mapped (we want a stable frequency for all VMs).
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync rc = RTTimerCreateEx(&pDevExt->pInvarTscRefineTimer, RT_NS_1SEC,
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync RTTIMER_FLAGS_CPU(RTMpCpuIdToSetIndex(pDevExt->idCpuInvarTscRefine)),
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync rc = RTTimerStart(pDevExt->pInvarTscRefineTimer, 2*RT_NS_100MS);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync if (rc == VERR_CPU_OFFLINE || rc == VERR_NOT_SUPPORTED)
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync rc = RTTimerCreateEx(&pDevExt->pInvarTscRefineTimer, RT_NS_1SEC, RTTIMER_FLAGS_CPU_ANY,
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync rc = RTTimerStart(pDevExt->pInvarTscRefineTimer, 2*RT_NS_100MS);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync OSDBGPRINT(("vboxdrv: Failed to create or start TSC frequency refinement timer: rc=%Rrc\n", rc));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @callback_method_impl{PFNRTMPWORKER,
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * RTMpOnSpecific callback for reading TSC and time on the CPU we started
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * the measurements on.}
87c5113417e917cdf64545d4f8e0a27047cea783vboxsyncDECLCALLBACK(void) supdrvGipInitReadTscAndNanoTsOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Measures the TSC frequency of the system.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * The TSC frequency can vary on systems which are not reported as invariant.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * On such systems the object of this function is to find out what the nominal,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * maximum TSC frequency under 'normal' CPU operation.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns VBox status code.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @param pDevExt Pointer to the device instance.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @param pGip Pointer to the GIP.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @param fRough Set if we're doing the rough calculation that the
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * TSC measuring code needs, where accuracy isn't all
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * that important (too high is better than to low).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * When clear we try for best accuracy that we can
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * achieve in reasonably short time.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsyncstatic int supdrvGipInitMeasureTscFreq(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, bool fRough)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint32_t nsTimerIncr = RTTimerGetSystemGranularity();
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync while (cTriesLeft-- > 0)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Synchronize with the host OS clock tick on systems without high
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * resolution time API (older Windows version for example).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Read the TSC and current time, noting which CPU we're on.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Delay for a while.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Sleep-wait since the TSC frequency is constant, it eases host load.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Shorter interval produces more variance in the frequency (esp. Windows).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint64_t msDelay = ( ((fRough ? 16 : 200) * RT_NS_1MS + nsTimerIncr - 1) / nsTimerIncr * nsTimerIncr - RT_NS_100US )
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync RTThreadSleep((RTMSINTERVAL)(msDelay - msElapsed));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Busy-wait keeping the frequency up.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Read the TSC and time again.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * If the CPU changes things get a bit complicated and what we
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * can get away with depends on the GIP mode / TSC reliablity.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync bool fDoXCall = false;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Synchronous TSC mode: we're probably fine as it's unlikely
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * that we were rescheduled because of TSC throttling or power
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * management reasons, so just go ahead.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /* Probably ok, maybe we should retry once?. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync Assert(pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_NOT_APPLICABLE);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * If we're just doing the rough measurement, do the cross call and
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * get on with things (we don't have deltas!).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Invariant TSC mode: It doesn't matter if we have delta available
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * for both CPUs. That is not something we can assume at this point.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Note! We cannot necessarily trust enmUseTscDelta here because it's
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * downgraded after each delta calculation and the delta
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * calculations may not be complete yet.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync else if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync/** @todo This section of code is never reached atm, consider dropping it later on... */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint32_t iStartCpuSet = RTMpCpuIdToSetIndex(idCpuStart);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint32_t iStopCpuSet = RTMpCpuIdToSetIndex(idCpuStop);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint16_t iStartGipCpu = iStartCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync ? pGip->aiCpuFromCpuSetIdx[iStartCpuSet] : UINT16_MAX;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint16_t iStopGipCpu = iStopCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync ? pGip->aiCpuFromCpuSetIdx[iStopCpuSet] : UINT16_MAX;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
874970279c503ed890db044712b8e5eaa6f6f82avboxsync if (RT_LIKELY(iStartTscDelta != INT64_MAX && iStopTscDelta != INT64_MAX))
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Invalid CPU indexes are not caused by online/offline races, so
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * we have to trigger driver load failure if that happens as GIP
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * and IPRT assumptions are busted on this system.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync else if (iStopGipCpu >= pGip->cCpus || iStartGipCpu >= pGip->cCpus)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync SUPR0Printf("vboxdrv: Unexpected CPU index in supdrvGipInitMeasureTscFreq.\n");
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync iStartCpuSet, iStartGipCpu, iStartTscDelta, iStopCpuSet, iStopGipCpu, iStopTscDelta);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * No valid deltas. We retry, if we're on our last retry
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * we do the cross call instead just to get a result. The
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * frequency will be refined in a few seconds anyways.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync else if (cTriesLeft > 0)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Asynchronous TSC mode: This is bad as the reason we usually
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * use this mode is to deal with variable TSC frequencies and
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * deltas. So, we need to get the TSC from the same CPU as
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * started it, we also need to keep that CPU busy. So, retry
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * and fall back to the cross call on the last attempt.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Try read the TSC and timestamp on the start CPU.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync int rc = RTMpOnSpecific(idCpuStart, supdrvGipInitReadTscAndNanoTsOnCpu, &uTscStop, &nsStop);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Calculate the TSC frequency and update it (shared with the refinement timer).
27fcebf3d224cda980acb78bc44349da8cde9d37vboxsync supdrvGipInitSetCpuFreq(pGip, nsStop - nsStart, uTscStop - uTscStart, 0);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Finds our (@a idCpu) entry, or allocates a new one if not found.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns Index of the CPU in the cache set.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pGip The GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The CPU ID.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic uint32_t supdrvGipFindOrAllocCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * ASSUMES that CPU IDs are constant.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync return i - 1;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * The calling CPU should be accounted as online, update GIP accordingly.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * This is used by supdrvGipCreate() as well as supdrvGipMpEvent().
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt The device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The CPU ID.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvGipMpEventOnlineOrInitOnCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Do this behind a spinlock with interrupts disabled as this can fire
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * on all CPUs simultaneously, see @bugref{6110}.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Update the globals.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU16(&pGip->cPresentCpus, RTMpGetPresentCount());
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU16(&pGip->cOnlineCpus, RTMpGetOnlineCount());
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Update the entry.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync u64NanoTS = RTTimeSystemNanoTS() - pGip->u32UpdateIntervalNS;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync i = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, pGip->u64CpuHz);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteS16(&pGip->aCPUs[i].iCpuSet, (int16_t)iCpuSet);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Update the APIC ID and CPU set index mappings.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU16(&pGip->aiCpuFromApicId[idApic], i);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU16(&pGip->aiCpuFromCpuSetIdx[iCpuSet], i);
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync /* Add this CPU to this set of CPUs we need to calculate the TSC-delta for. */
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, RTMpCpuIdToSetIndex(idCpu));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Update the Mp online/offline counter. */
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync /* Commit it. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE);
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * RTMpOnSpecific callback wrapper for supdrvGipMpEventOnlineOrInitOnCpu().
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * @param idCpu The CPU ID we are running on.
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * @param pvUser1 Opaque pointer to the device instance data.
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * @param pvUser2 Not used.
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsyncstatic DECLCALLBACK(void) supdrvGipMpEventOnlineCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * The CPU should be accounted as offline, update the GIP accordingly.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * This is used by supdrvGipMpEvent.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt The device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The CPU ID.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvGipMpEventOffline(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync unsigned i;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Update the Mp online/offline counter. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Reset the TSC delta, we will recalculate it lazily. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteS64(&pGip->aCPUs[i].i64TSCDelta, INT64_MAX);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Remove this CPU from the set of CPUs that we have obtained the TSC deltas. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync RTCpuSetDelByIndex(&pDevExt->TscDeltaObtainedCpuSet, iCpuSet);
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync /* Commit it. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_OFFLINE);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Multiprocessor event notification callback.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * This is used to make sure that the GIP master gets passed on to
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * another CPU. It also updates the associated CPU data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param enmEvent The event.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The cpu it applies to.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser Pointer to the device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic DECLCALLBACK(void) supdrvGipMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync RTMpOnSpecific(idCpu, supdrvGipMpEventOnlineCallback, pDevExt, NULL /* pvUser2 */);
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * Recompute TSC-delta for the newly online'd CPU.
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync supdrvTscDeltaThreadStartMeasurement(pDevExt, false /* fForceAll */);
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync uint32_t iCpu = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Make sure there is a master GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync RTCPUID idGipMaster = ASMAtomicReadU32(&pDevExt->idGipMaster);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * The GIP master is going offline, find a new one.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync unsigned i;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync for (i = 0; i < RTCPUSET_MAX_CPUS; i++)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync Log(("supdrvGipMpEvent: Gip master %#lx -> %#lx\n", (long)idGipMaster, (long)idNewGipMaster));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicCmpXchgSize(&pDevExt->idGipMaster, idNewGipMaster, idGipMaster, fIgnored);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * On CPU initialization callback for RTMpOnAll.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The CPU ID.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser1 The device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser2 The GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic DECLCALLBACK(void) supdrvGipInitOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* This is good enough, even though it will update some of the globals a
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync bit to much. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync supdrvGipMpEventOnlineOrInitOnCpu((PSUPDRVDEVEXT)pvUser1, idCpu);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Callback used by supdrvDetermineAsyncTSC to read the TSC on a CPU.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu Ignored.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser1 Where to put the TSC.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser2 Ignored.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic DECLCALLBACK(void) supdrvGipInitDetermineAsyncTscWorker(RTCPUID idCpu, void *pvUser1, void *pvUser2)
b0deb52d71d46987430324c2e39de46a2791a738vboxsync Assert(RTMpCpuIdToSetIndex(idCpu) == (intptr_t)pvUser2);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU64((uint64_t volatile *)pvUser1, ASMReadTSC());
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Determine if Async GIP mode is required because of TSC drift.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * When using the default/normal timer code it is essential that the time stamp counter
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * (TSC) runs never backwards, that is, a read operation to the counter should return
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * a bigger value than any previous read operation. This is guaranteed by the latest
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * AMD CPUs and by newer Intel CPUs which never enter the C2 state (P4). In any other
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * case we have to choose the asynchronous timer mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param poffMin Pointer to the determined difference between different
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * cores (optional, can be NULL).
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @return false if the time stamp counters appear to be synchronized, true otherwise.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic bool supdrvGipInitDetermineAsyncTsc(uint64_t *poffMin)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Just iterate all the cpus 8 times and make sure that the TSC is
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * ever increasing. We don't bother taking TSC rollover into account.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync bool fAsync = false;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync while (cLoops-- > 0)
b0deb52d71d46987430324c2e39de46a2791a738vboxsync rc = RTMpOnSpecific(RTMpCpuIdFromSetIndex(iCpu), supdrvGipInitDetermineAsyncTscWorker,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync Log(("supdrvGipInitDetermineAsyncTsc: iCpu=%d cLoops=%d CurTsc=%llx PrevTsc=%llx\n",
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Gather statistics (except the first time). */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync AssertMsg(rc == VERR_CPU_NOT_FOUND || rc == VERR_CPU_OFFLINE, ("%d\n", rc));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* broke out of the loop. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync *poffMin = offMin; /* Almost RTMpOnSpecific profiling. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync Log(("supdrvGipInitDetermineAsyncTsc: returns %d; iEndCpu=%d rc=%d offMin=%llx offMax=%llx\n",
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync#if !defined(RT_OS_SOLARIS) && !defined(RT_OS_OS2) && !defined(RT_OS_WINDOWS)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("vboxdrv: fAsync=%d offMin=%#lx offMax=%#lx\n", fAsync, (long)offMin, (long)offMax));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * supdrvGipInit() worker that determines the GIP TSC mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns The most suitable TSC mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt Pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic SUPGIPMODE supdrvGipInitDetermineTscMode(PSUPDRVDEVEXT pDevExt)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Establish whether the CPU advertises TSC as invariant, we need that in
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * a couple of places below.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (ASMIsValidExtRange(uEAX) && uEAX >= 0x80000007)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * On single CPU systems, we don't need to consider ASYNC mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync return fInvariantTsc ? SUPGIPMODE_INVARIANT_TSC : SUPGIPMODE_SYNC_TSC;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Allow the user and/or OS specific bits to force async mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Use invariant mode if the CPU says TSC is invariant.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * TSC is not invariant and we're on SMP, this presents two problems:
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * (1) There might be a skew between the CPU, so that cpu0
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * returns a TSC that is slightly different from cpu1.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * This screw may be due to (2), bad TSC initialization
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * or slightly different TSC rates.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * (2) Power management (and other things) may cause the TSC
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * to run at a non-constant speed, and cause the speed
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * to be different on the cpus. This will result in (1).
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * If any of the above is detected, we will have to use ASYNC mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* (1). Try check for current differences between the cpus. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (supdrvGipInitDetermineAsyncTsc(&u64DiffCoresIgnored))
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* (2) If it's an AMD CPU with power management, we won't trust its TSC. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Check for APM support. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (ASMIsValidExtRange(uEAX) && uEAX >= 0x80000007)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (uEDX & 0x3e) /* STC|TM|THERMTRIP|VID|FID. Ignore TS. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Initializes per-CPU GIP information.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pGip Pointer to the GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pCpu Pointer to which GIP CPU to initalize.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64NanoTS The current nanosecond timestamp.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @param uCpuHz The CPU frequency to set, 0 if the caller doesn't know.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsyncstatic void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pCpu->i64TSCDelta = pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED ? INT64_MAX : 0;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteSize(&pCpu->enmState, SUPGIPCPUSTATE_INVALID);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * The first time we're called, we don't have a CPU frequency handy,
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * so pretend it's a 4 GHz CPU. On CPUs that are online, we'll get
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * called again and at that point we have a more plausible CPU frequency
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * value handy. The frequency history will also be adjusted again on
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * the 2nd timer callout (maybe we can skip that now?).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync pCpu->u32UpdateIntervalTSC = (uint32_t)((_4G - 1) / pGip->u32UpdateHz);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync pCpu->u32UpdateIntervalTSC = (uint32_t)(uCpuHz / pGip->u32UpdateHz);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Initializes the GIP data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt Pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pGip Pointer to the read-write kernel mapping of the GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param HCPhys The physical address of the GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64NanoTS The current nanosecond timestamp.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param uUpdateHz The update frequency.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param uUpdateIntervalNS The update interval in nanoseconds.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param cCpus The CPU count.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS, unsigned cCpus)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync size_t const cbGip = RT_ALIGN_Z(RT_OFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), PAGE_SIZE);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync unsigned i;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync LogFlow(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Initialize the structure.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pGip->u32Mode = supdrvGipInitDetermineTscMode(pDevExt);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /*|| pGip->u32Mode == SUPGIPMODE_SYNC_TSC */)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pGip->enmUseTscDelta = supdrvOSAreTscDeltasInSync() /* Allow OS override (windows). */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ? SUPGIPUSETSCDELTA_ZERO_CLAIMED : SUPGIPUSETSCDELTA_PRACTICALLY_ZERO /* downgrade later */;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pGip->enmUseTscDelta = SUPGIPUSETSCDELTA_NOT_APPLICABLE;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromApicId); i++)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx); i++)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync for (i = 0; i < cCpus; i++)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, 0 /*uCpuHz*/);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Link it to the device extension.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Creates the GIP.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns VBox status code.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pDevExt Instance data. GIP stuff may be updated.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * Assert order.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync Assert(pDevExt->u32SystemTimerGranularityGrant == 0);
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync Assert(pDevExt->mtxTscDelta != NIL_RTSEMFASTMUTEX);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Check the CPU count.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync || cCpus > 256 /* ApicId is used for the mappings */)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync SUPR0Printf("VBoxDrv: Too many CPUs (%u) for the GIP (max %u)\n", cCpus, RT_MIN(RTCPUSET_MAX_CPUS, 256));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Allocate a contiguous set of pages with a default kernel mapping.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), false /*fExecutable*/);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync OSDBGPRINT(("supdrvGipCreate: failed to allocate the GIP page. rc=%d\n", rc));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pGip = (PSUPGLOBALINFOPAGE)RTR0MemObjAddress(pDevExt->GipMemObj); AssertPtr(pGip);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync HCPhysGip = RTR0MemObjGetPagePhysAddr(pDevExt->GipMemObj, 0); Assert(HCPhysGip != NIL_RTHCPHYS);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Find a reasonable update interval and initialize the structure.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /** @todo figure out why using a 100Ms interval upsets timekeeping in VMs.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * See @bugref{6710}. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync u32SystemResolution = RTTimerGetSystemGranularity();
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /*=Hz*/, u32Interval, cCpus);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Important sanity check...
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (RT_UNLIKELY( pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_ZERO_CLAIMED
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync OSDBGPRINT(("supdrvGipCreate: Host-OS/user claims the TSC-deltas are zero but we detected async. TSC! Bad.\n"));
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync /* It doesn't make sense to do TSC-delta detection on systems we detect as async. */
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync AssertReturn( pGip->u32Mode != SUPGIPMODE_ASYNC_TSC
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync || pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED, VERR_INTERNAL_ERROR_3);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Do the TSC frequency measurements.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * If we're in invariant TSC mode, just to a quick preliminary measurement
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * that the TSC-delta measurement code can use to yield cross calls.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * If we're in any of the other two modes, neither which require MP init,
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * notifications or deltas for the job, do the full measurement now so
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * that supdrvGipInitOnCpu() can populate the TSC interval and history
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * array with more reasonable values.
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync rc = supdrvGipInitMeasureTscFreq(pDevExt, pGip, true /*fRough*/); /* cannot fail */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync supdrvGipInitStartTimerForRefiningInvariantTscFreq(pDevExt, pGip);
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync rc = supdrvGipInitMeasureTscFreq(pDevExt, pGip, false /*fRough*/);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Start TSC-delta measurement thread before we start getting MP
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * events that will try kick it into action (includes the
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Do GIP initialization on all online CPUs. Wake up the
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * TSC-delta thread afterwards.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Measure the TSC deltas now that we have MP notifications.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync } while (--cTries > 0);
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Create the timer.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * If CPU_ALL isn't supported we'll have to fall back to synchronous mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n"));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * We're good.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync Log(("supdrvGipCreate: %u ns interval.\n", u32Interval));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync OSDBGPRINT(("supdrvGipCreate: supdrvMeasureInitialTscDeltas failed. rc=%Rrc\n", rc));
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /* Releases timer frequency increase too. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Invalidates the GIP data upon termination.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pGip Pointer to the read-write kernel mapping of the GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync unsigned i;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Terminates the GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt Instance data. GIP stuff may be updated.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncvoid VBOXCALL supdrvGipDestroy(PSUPDRVDEVEXT pDevExt)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvGipDestroy: pDevExt=%p pGip=%p pGipTimer=%p GipMemObj=%p\n", pDevExt,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pDevExt->GipMemObj != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pDevExt->GipMemObj) : NULL,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Stop receiving MP notifications before tearing anything else down.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync RTMpNotificationDeregister(supdrvGipMpEvent, pDevExt);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Terminate the TSC-delta measurement thread and resources.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Destroy the TSC-refinement timer.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Invalid the GIP data.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Destroy the timer and free the GIP memory object.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTTimerDestroy(pDevExt->pGipTimer); AssertRC(rc);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = RTR0MemObjFree(pDevExt->GipMemObj, true /* free mappings */); AssertRC(rc);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Finally, make sure we've release the system timer resolution request
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * if one actually succeeded and is still pending.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Update Timer Related Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Update Timer Related Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * GIP Update Timer Related Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Worker routine for supdrvGipUpdate() and supdrvGipUpdatePerCpu() that
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * updates all the per cpu data except the transaction id.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt The device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pGipCpu Pointer to the per cpu data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64NanoTS The current time stamp.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64TSC The current TSC.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param iTick The current timer tick.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @remarks Can be called with interrupts disabled!
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvGipDoUpdateCpu(PSUPDRVDEVEXT pDevExt, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS, uint64_t u64TSC, uint64_t iTick)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Delta between this and the previous update. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicUoWriteU32(&pGipCpu->u32PrevUpdateIntervalNS, (uint32_t)(u64NanoTS - pGipCpu->u64NanoTS));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Update the NanoTS.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Calc TSC delta.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * We don't need to keep realculating the frequency when it's invariant, so
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * the remainder of this function is only for the sync and async TSC modes.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * On the 2nd and 3rd callout, reset the history with the current TSC
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * interval since the values entered by supdrvGipInit are totally off.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * The interval on the 1st callout completely unreliable, the 2nd is a bit
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * better, while the 3rd should be most reliable.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync /** @todo Could we drop this now that we initializes the history
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * with nominal TSC frequency values? */
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync unsigned i;
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync for (i = 0; i < RT_ELEMENTS(pGipCpu->au32TSCHistory); i++)
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync ASMAtomicUoWriteU32(&pGipCpu->au32TSCHistory[i], (uint32_t)u64TSCDelta);
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * Validate the NanoTS deltas between timer fires with an arbitrary threshold of 0.5%.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * Wait until we have at least one full history since the above history reset. The
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * assumption is that the majority of the previous history values will be tolerable.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * See @bugref{6710} comment #67.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync /** @todo Could we drop the fuding there now that we initializes the history
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * with nominal TSC frequency values? */
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync uint32_t uNanoTsThreshold = pGip->u32UpdateIntervalNS / 200;
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync if ( pGipCpu->u32PrevUpdateIntervalNS > pGip->u32UpdateIntervalNS + uNanoTsThreshold
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync || pGipCpu->u32PrevUpdateIntervalNS < pGip->u32UpdateIntervalNS - uNanoTsThreshold)
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * TSC History.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync iTSCHistoryHead = (pGipCpu->iTSCHistoryHead + 1) & 7;
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync ASMAtomicWriteU32(&pGipCpu->iTSCHistoryHead, iTSCHistoryHead);
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync ASMAtomicWriteU32(&pGipCpu->au32TSCHistory[iTSCHistoryHead], (uint32_t)u64TSCDelta);
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * UpdateIntervalTSC = average of last 8,2,1 intervals depending on update HZ.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * On Windows, we have an occasional (but recurring) sour value that messed up
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync * the history but taking only 1 interval reduces the precision overall.
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[5];
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[6];
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[7];
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync /* Value chosen for a 2GHz Athlon64 running linux 2.6.10/11. */
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 14;
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[(iTSCHistoryHead - 1) & 7];
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync /* value chosen on a 2GHz thinkpad running windows */
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 7;
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync /* This value hasn't be checked yet.. waiting for OS/2 and 33Hz timers.. :-) */
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 6;
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync ASMAtomicWriteU32(&pGipCpu->u32UpdateIntervalTSC, u32UpdateIntervalTSC + u32UpdateIntervalTSCSlack);
8689d10cc9774b5c64bbfcf02b8ae32e4c321aa6vboxsync u64CpuHz = ASMMult2xU32RetU64(u32UpdateIntervalTSC, RT_NS_1SEC);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Updates the GIP.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt The device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64NanoTS The current nanosecond timesamp.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64TSC The current TSC timesamp.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The CPU ID.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param iTick The current timer tick.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @remarks Can be called with interrupts disabled!
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvGipUpdate(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC, RTCPUID idCpu, uint64_t iTick)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Determine the relevant CPU data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync unsigned iCpu = pGip->aiCpuFromApicId[ASMGetApicId()];
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Start update transaction.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* this can happen on win32 if we're taking to long and there are more CPUs around. shouldn't happen though. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Recalc the update frequency every 0x800th time.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if ( pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC /* cuz we're not recalculating the frequency on invariants hosts. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync && !(pGipCpu->u32TransactionId & (GIP_UPDATEHZ_RECALC_FREQ * 2 - 2)))
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync#ifdef RT_ARCH_AMD64 /** @todo fix 64-bit div here to work on x86 linux. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync uint64_t u64Delta = u64NanoTS - pGip->u64NanoTSLastUpdateHz;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync uint32_t u32UpdateHz = (uint32_t)((RT_NS_1SEC_64 * GIP_UPDATEHZ_RECALC_FREQ) / u64Delta);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /** @todo r=ramshankar: Changing u32UpdateHz might screw up TSC frequency
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * calculation on non-invariant hosts if it changes the history decision
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * taken in supdrvGipDoUpdateCpu(). */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync uint64_t u64Interval = u64Delta / GIP_UPDATEHZ_RECALC_FREQ;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU32(&pGip->u32UpdateHz, u32UpdateHz);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU32(&pGip->u32UpdateIntervalNS, (uint32_t)u64Interval);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteU64(&pGip->u64NanoTSLastUpdateHz, u64NanoTS | 1);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Update the data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Complete transaction.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Updates the per cpu GIP data for the calling cpu.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt The device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64NanoTS The current nanosecond timesamp.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param u64TSC The current TSC timesamp.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idCpu The CPU ID.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param idApic The APIC id for the CPU index.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param iTick The current timer tick.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @remarks Can be called with interrupts disabled!
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvGipUpdatePerCpu(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Avoid a potential race when a CPU online notification doesn't fire on
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * the onlined CPU but the tick creeps in before the event notification is
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync iCpu = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (pGip->aCPUs[iCpu].enmState == SUPGIPCPUSTATE_OFFLINE)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Start update transaction.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Update the data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Complete transaction.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Timer callback function for the sync and invariant GIP modes.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pTimer The timer.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser Opaque pointer to the device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param iTick The timer tick.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync RTCCUINTREG fOldFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * The calculations in supdrvGipUpdate() is somewhat timing sensitive,
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * missing timer ticks is not an option for GIP because the GIP users
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * will end up incrementing the time in 1ns per time getter call until
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * there is a complete timer update. So, if the delta has yet to be
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * calculated, we just pretend it is zero for now (the GIP users
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * probably won't have it for a wee while either and will do the same).
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * We could maybe on some platforms try cross calling a CPU with a
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * working delta here, but it's not worth the hassle since the
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * likelyhood of this happening is really low. On Windows, Linux, and
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * Solaris timers fire on the CPU they were registered/started on.
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * Darwin timers doesn't necessarily (they are high priority threads).
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync uint32_t iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync uint16_t iGipCpu = RT_LIKELY(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync int64_t iTscDelta = pGip->aCPUs[iGipCpu].i64TSCDelta;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync supdrvGipUpdate(pDevExt, u64NanoTS, u64TSC, NIL_RTCPUID, iTick);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Timer callback function for async GIP mode.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pTimer The timer.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser Opaque pointer to the device extension.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param iTick The timer tick.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync RTCCUINTREG fOldFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /** @todo reset the transaction number and whatnot when iTick == 1. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync supdrvGipUpdate(pDevExt, NanoTS, u64TSC, idCpu, iTick);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync supdrvGipUpdatePerCpu(pDevExt, NanoTS, u64TSC, idCpu, ASMGetApicId(), iTick);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * TSC Delta Measurements And Related Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * TSC Delta Measurements And Related Code
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * TSC Delta Measurements And Related Code
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Select TSC delta measurement algorithm.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync/** For padding variables to keep them away from other cache lines. Better too
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * large than too small!
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * @remarks Current AMD64 and x86 CPUs seems to use 64 bytes. There are claims
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * that NetBurst had 128 byte cache lines while the 486 thru Pentium
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * III had 32 bytes cache lines. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * TSC delta measurment algorithm \#2 result entry.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * TSC delta measurment algorithm \#2 Data.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** Padding to make sure the iCurSeqNo is in its own cache line. */
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /** The current sequence number of this worker. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** Padding to make sure the iCurSeqNo is in its own cache line. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint32_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint32_t) - 1];
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /** Result table. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync/** Pointer to the data for TSC delta mesurment algorithm \#2 .*/
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsynctypedef SUPDRVTSCDELTAMETHOD2 *PSUPDRVTSCDELTAMETHOD2;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * The TSC delta synchronization struct, version 2.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * The syncrhonization variable is completely isolated in its own cache line
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * (provided our max cache line size estimate is correct).
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** Padding to make sure the uVar1 is in its own cache line. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** The synchronization variable, holds values GIP_TSC_DELTA_SYNC_*. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync /** Sequence synchronizing variable used for post 'GO' synchronization. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** Padding to make sure the uVar1 is in its own cache line. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync uint64_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t) - 2];
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync /** Start RDTSC value. Put here mainly to save stack space. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync /** Copy of SUPDRVGIPTSCDELTARGS::cMaxTscTicks. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsyncAssertCompileSize(SUPTSCDELTASYNC2, GIP_TSC_DELTA_CACHE_LINE_SIZE * 2 + sizeof(uint64_t));
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync/** Prestart wait. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync#define GIP_TSC_DELTA_SYNC2_PRESTART_WAIT UINT32_C(0x0ffe)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync/** Prestart aborted. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync#define GIP_TSC_DELTA_SYNC2_PRESTART_ABORT UINT32_C(0x0fff)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync/** Ready (on your mark). */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync/** Steady (get set). */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync/** Used by the verfication test. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync/** We reached the time limit. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync#define GIP_TSC_DELTA_SYNC2_TIMEOUT UINT32_C(0x1ffe)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync/** The other party won't touch the sync struct ever again. */
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * Argument package/state passed by supdrvMeasureTscDeltaOne() to the RTMpOn
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * callback worker.
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync * @todo add
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** The device extension. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** Pointer to the GIP CPU array entry for the worker. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync /** Pointer to the GIP CPU array entry for the master. */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync /** The maximum number of ticks to spend in supdrvMeasureTscDeltaCallback.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * (This is what we need a rough TSC frequency for.) */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync /** Used to abort synchronization setup. */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync bool volatile fAbortSetup;
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** Padding to make sure the master variables live in its own cache lines. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
b9efc54979b7ee83352754810d59d75543b474a1vboxsync /** @name Master
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync /** The time the master spent in the MP worker. */
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync /** The iTry value when stopped at. */
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync /** Set if the run timed out. */
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync bool volatile fTimedOut;
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** Pointer to the master's synchronization struct (on stack). */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Master data union. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Data (master) for delta verification. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Verification test TSC values for the master. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Data (master) for measurement method \#2. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Data and sequence number. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** The lag setting for the next run. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Number of hits. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** The verifier verdict, VINF_SUCCESS if ok, VERR_OUT_OF_RANGE if not,
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * VERR_TRY_AGAIN on timeout. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** The maximum difference between TSC read during delta verification. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** The minimum difference between two TSC reads during verification. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** The bad TSC diff, worker relative to master (= worker - master).
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * Negative value means the worker is behind the master. */
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync /** Padding to make sure the worker variables live is in its own cache line. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync uint64_t au64CacheLinePaddingBetween[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
b9efc54979b7ee83352754810d59d75543b474a1vboxsync /** @name Proletarian
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** Pointer to the worker's synchronization struct (on stack). */
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync /** The time the worker spent in the MP worker. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Worker data union. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Data (worker) for delta verification. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Verification test TSC values for the worker. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Data (worker) for measurement method \#2. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** Data and sequence number. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** The lag setting for the next run (set by master). */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /** Padding to make sure the above is in its own cache line. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync uint64_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsynctypedef SUPDRVGIPTSCDELTARGS *PSUPDRVGIPTSCDELTARGS;
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync/** @name Macros that implements the basic synchronization steps common to
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * the algorithms.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * Must be used from loop as the timeouts are implemented via 'break' statements
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * at the moment.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync#if defined(DEBUG_bird) /* || defined(VBOX_STRICT) */
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync# define TSCDELTA_DBG_START_LOOP() do { iDbgCounter = 0; } while (0)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync do { iDbgCounter++; if ((iDbgCounter & UINT32_C(0x01ffffff)) == 0) RT_BREAKPOINT(); } while (0)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync# define TSCDELTA_DBG_VARS() ((void)0)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync# define TSCDELTA_DBG_START_LOOP() ((void)0)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync# define TSCDELTA_DBG_CHECK_LOOP() ((void)0)
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync# define TSCDELTA_DBG_SYNC_MSG(a_Args) SUPR0Printf a_Args
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync# define TSCDELTA_DBG_SYNC_MSG2(a_Args) SUPR0Printf a_Args
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync# define TSCDELTA_DBG_SYNC_MSG9(a_Args) SUPR0Printf a_Args
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsyncstatic bool supdrvTscDeltaSync2_Before(PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync bool fIsMaster, PRTCCUINTREG pfEFlags, PSUPDRVGIPTSCDELTARGS pArgs)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync uint32_t const iMaxSeq = iMySeq + 16; /* For the last loop, darn linux/freebsd C-ishness. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync *pfEFlags = X86_EFL_IF | X86_EFL_1; /* should shut up most nagging compilers. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * The master tells the worker to get on it's mark.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync { /* likely*/ }
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #1 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return false;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * Wait for the on your mark signal (ack in the master case). We process timeouts here.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync /* Abort? */
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #2 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync return false;
357175321a45346fc98b70f7644399f412ee9c3evboxsync /* Check for timeouts every so often (not every loop in case RDTSC is
357175321a45346fc98b70f7644399f412ee9c3evboxsync trapping or something). Must check the first time around. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync#if 0 /* For debugging the timeout paths. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync && ASMReadTSC() - pMySync->uTscStart > pMySync->cMaxTscTicks)
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync#if 0 /* This is crazy, I know, but enable this code and the results are markedly better when enabled on the 1.4GHz AMD (debug). */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync /* Try switch our own state into timeout mode so the master cannot tell us to 'GO',
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync ignore the timeout if we've got the go ahead already (simpler). */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (ASMAtomicCmpXchgU32(&pMySync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_READY))
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: timeout\n", fIsMaster ? "master" : "worker"));
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_STEADY);
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return false;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * Interrupts are now disabled and will remain disabled until we do
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * TSCDELTA_MASTER_SYNC_AFTER / TSCDELTA_OTHER_SYNC_AFTER.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * The worker tells the master that it is on its mark and that the master
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * need to get into position as well.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync { /* likely */ }
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #3 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return false;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * The master sends the 'go' to the worker and wait for ACK.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync { /* likely */ }
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #4 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return false;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * Wait for the 'go' signal (ack in the master case).
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (RT_LIKELY(u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY))
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync { /* likely */ }
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #5 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return false;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * The worker acks the 'go' (shouldn't fail).
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync { /* likely */ }
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #6 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return false;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * Try enter mostly lockstep execution with it.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync iOtherSeq1 = ASMAtomicXchgU32(&pOtherSync->uSyncSeq, iMySeq);
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return true;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync /* Did the other guy give up? Should we give up? */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return true;
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync return true;
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync#define TSCDELTA_MASTER_SYNC_BEFORE(a_pMySync, a_pOtherSync, a_pfEFlags, a_pArgs) \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, true /*fIsMaster*/, a_pfEFlags, a_pArgs))) \
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync { /*likely*/ } \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync else if (true) \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG9(("sync/before/master: #89\n")); \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync } else do {} while (0)
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync#define TSCDELTA_OTHER_SYNC_BEFORE(a_pMySync, a_pOtherSync, a_pfEFlags, a_pArgs) \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, false /*fIsMaster*/, a_pfEFlags, a_pArgs))) \
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync { /*likely*/ } \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync else if (true) \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG9(("sync/before/other: #89\n")); \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync } else do {} while (0)
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsyncstatic bool supdrvTscDeltaSync2_After(PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * Wait for the 'ready' signal. In the master's case, this means the
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * worker has completed its data collection, while in the worker's case it
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * means the master is done processing the data and it's time for the next
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * loop iteration (or whatever).
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync uint32_t u32Tmp = ASMAtomicReadU32(&pMySync->uSyncVar);
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync || (u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY && !fIsMaster) /* kicked twice => race */ )
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync return true;
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync { /* likely */}
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/after/other: #1 u32Tmp=%#x\n", u32Tmp));
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync return false; /* shouldn't ever happen! */
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync#define TSCDELTA_MASTER_SYNC_AFTER(a_pMySync, a_pOtherSync, a_fEFlags) \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync if (RT_LIKELY(supdrvTscDeltaSync2_After(a_pMySync, a_pOtherSync, true /*fIsMaster*/, a_fEFlags))) \
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync { /* likely */ } \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync else if (true) \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG9(("sync/after/master: #97\n")); \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync } else do {} while (0)
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync#define TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(a_pMySync, a_pOtherSync) \
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync * Tell the worker that we're done processing the data and ready for the next round. \
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync { /* likely */ } \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync else if (true)\
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/after/master: #99 uSyncVar=%#x\n", (a_pOtherSync)->uSyncVar)); \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync } else do {} while (0)
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync#define TSCDELTA_OTHER_SYNC_AFTER(a_pMySync, a_pOtherSync, a_fEFlags) \
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync if (true) { \
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * Tell the master that we're done collecting data and wait for the next round to start. \
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync { /* likely */ } \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/after/other: #0 uSyncVar=%#x\n", (a_pOtherSync)->uSyncVar)); \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync if (RT_LIKELY(supdrvTscDeltaSync2_After(a_pMySync, a_pOtherSync, false /*fIsMaster*/, a_fEFlags))) \
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync { /* likely */ } \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG9(("sync/after/other: #98\n")); \
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync } else do {} while (0)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * TSC delta measurment algorithm \#1 (GIP_TSC_DELTA_METHOD_1).
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * We ignore the first few runs of the loop in order to prime the
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * cache. Also, we need to be careful about using 'pause' instruction
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * in critical busy-wait loops in this code - it can cause undesired
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * behaviour with hyperthreading.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * We try to minimize the measurement error by computing the minimum
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * read time of the compare statement in the worker by taking TSC
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * measurements across it.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * It must be noted that the computed minimum read time is mostly to
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * eliminate huge deltas when the worker is too early and doesn't by
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * itself help produce more accurate deltas. We allow two times the
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * computed minimum as an arbibtrary acceptable threshold. Therefore,
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * it is still possible to get negative deltas where there are none
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * when the worker is earlier. As long as these occasional negative
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * deltas are lower than the time it takes to exit guest-context and
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * the OS to reschedule EMT on a different CPU we won't expose a TSC
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * that jumped backwards. It is because of the existence of the
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * negative deltas we don't recompute the delta with the master and
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * worker interchanged to eliminate the remaining measurement error.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * @param pArgs The argument/state data.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * @param pMySync My synchronization structure.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * @param pOtherSync My partner's synchronization structure.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * @param fIsMaster Set if master, clear if worker.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * @param iTry The attempt number.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsyncstatic void supdrvTscDeltaMethod1Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync for (iLoop = 0; iLoop < GIP_TSC_DELTA_LOOPS; iLoop++)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * The master.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync AssertMsg(pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD,
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ("%#llx idMaster=%#x idWorker=%#x (idGipMaster=%#x)\n",
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync pGipCpuMaster->u64TSCSample, pGipCpuMaster->idCpu, pGipCpuWorker->idCpu, pArgs->pDevExt->idGipMaster));
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, ASMReadTSC());
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync } while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync /* Process the data. */
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync if (pGipCpuWorker->u64TSCSample != GIP_TSC_DELTA_RSVD)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync - (pGipCpuMaster->u64TSCSample - pGipCpuMaster->i64TSCDelta);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync : iDelta > pGipCpuWorker->i64TSCDelta || pGipCpuWorker->i64TSCDelta == INT64_MAX)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync /* Reset our TSC sample and tell the worker to move on. */
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, GIP_TSC_DELTA_RSVD);
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * The worker.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ASMAtomicReadU64(&pGipCpuMaster->u64TSCSample); /* Warm the cache line. */
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * Keep reading the TSC until we notice that the master has read his. Reading
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * the TSC -after- the master has updated the memory is way too late. We thus
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * compensate by trying to measure how long it took for the worker to notice
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * the memory flushed from the master.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync } while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync /* This is totally arbitrary a.k.a I don't like it but I have no better ideas for now. */
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, uTscWorker);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG9(("sync/method1loop/%s: #92 iLoop=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iLoop,
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * We must reset the worker TSC sample value in case it gets picked as a
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * GIP master later on (it's trashed above, naturally).
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync#endif /* GIP_TSC_DELTA_METHOD_1 */
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * TSC delta measurement algorithm \#2 configuration and code - Experimental!!
8f507027ddf8199099a107a263964ec7560b1a9evboxsync# define GIP_TSC_DELTA_M2_LOOPS (7 + GIP_TSC_DELTA_M2_PRIMER_LOOPS)
dc45a8f3e936581748c248e00ce572cfe3ea331evboxsyncstatic void supdrvTscDeltaMethod2ProcessDataOnMaster(PSUPDRVGIPTSCDELTARGS pArgs, uint32_t iLoop)
b9efc54979b7ee83352754810d59d75543b474a1vboxsync int64_t iMasterTscDelta = pArgs->pMaster->i64TSCDelta;
dc45a8f3e936581748c248e00ce572cfe3ea331evboxsync * Look for matching entries in the master and worker tables.
b9efc54979b7ee83352754810d59d75543b474a1vboxsync for (idxResult = 0; idxResult < RT_ELEMENTS(pArgs->uMaster.M2.Data.aResults); idxResult++)
b9efc54979b7ee83352754810d59d75543b474a1vboxsync uint32_t idxOther = pArgs->uMaster.M2.Data.aResults[idxResult].iSeqOther;
b9efc54979b7ee83352754810d59d75543b474a1vboxsync if (idxOther < RT_ELEMENTS(pArgs->uWorker.M2.Data.aResults))
b9efc54979b7ee83352754810d59d75543b474a1vboxsync if (pArgs->uWorker.M2.Data.aResults[idxOther].iSeqOther == pArgs->uMaster.M2.Data.aResults[idxResult].iSeqMine)
b9efc54979b7ee83352754810d59d75543b474a1vboxsync iDelta = pArgs->uWorker.M2.Data.aResults[idxOther].uTsc
b9efc54979b7ee83352754810d59d75543b474a1vboxsync - (pArgs->uMaster.M2.Data.aResults[idxResult].uTsc - iMasterTscDelta);
dc45a8f3e936581748c248e00ce572cfe3ea331evboxsync * Save the results.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * The core function of the 2nd TSC delta mesurment algorithm.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * The idea here is that we have the two CPUs execute the exact same code
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * collecting a largish set of TSC samples. The code has one data dependency on
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * the other CPU which intention it is to synchronize the execution as well as
81c6115ff3bb02e166ee8f762d30c4ba5e3db08avboxsync * help cross references the two sets of TSC samples (the sequence numbers).
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * The @a fLag parameter is used to modify the execution a tiny bit on one or
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * both of the CPUs. When @a fLag differs between the CPUs, it is thought that
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * it will help with making the CPUs enter lock step execution occationally.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsyncstatic void supdrvTscDeltaMethod2CollectData(PSUPDRVTSCDELTAMETHOD2 pMyData, uint32_t volatile *piOtherSeqNo, bool fLag)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync SUPDRVTSCDELTAMETHOD2ENTRY *pEntry = &pMyData->aResults[0];
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync while (cLeft-- > 0)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync uint32_t iSeqMine = ASMAtomicIncU32(&pMyData->iCurSeqNo);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync uint32_t iSeqOther = ASMAtomicReadU32(piOtherSeqNo);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync ASMSerializeInstruction(); /* Way better result than with ASMMemoryFenceSSE2() in this position! */
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * TSC delta measurment algorithm \#2 (GIP_TSC_DELTA_METHOD_2).
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * See supdrvTscDeltaMethod2CollectData for algorithm details.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * @param pArgs The argument/state data.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * @param pMySync My synchronization structure.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * @param pOtherSync My partner's synchronization structure.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * @param fIsMaster Set if master, clear if worker.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * @param iTry The attempt number.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsyncstatic void supdrvTscDeltaMethod2Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
dc45a8f3e936581748c248e00ce572cfe3ea331evboxsync for (iLoop = 0; iLoop < GIP_TSC_DELTA_M2_LOOPS; iLoop++)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * Adjust the loop lag fudge.
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync /* Lag during the priming to be nice to everyone.. */
dc45a8f3e936581748c248e00ce572cfe3ea331evboxsync if (iLoop < (GIP_TSC_DELTA_M2_LOOPS - GIP_TSC_DELTA_M2_PRIMER_LOOPS) / 4)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync /* 25 % of the body without lagging. */
dc45a8f3e936581748c248e00ce572cfe3ea331evboxsync else if (iLoop < (GIP_TSC_DELTA_M2_LOOPS - GIP_TSC_DELTA_M2_PRIMER_LOOPS) / 4 * 2)
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync /* 25 % of the body with both lagging. */
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync /* 50% of the body with alternating lag. */
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * Sync up with the worker and collect data.
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync supdrvTscDeltaMethod2CollectData(&pArgs->uMaster.M2.Data, &pArgs->uWorker.M2.Data.iCurSeqNo, pArgs->uMaster.M2.fLag);
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * Process the data.
dc45a8f3e936581748c248e00ce572cfe3ea331evboxsync supdrvTscDeltaMethod2ProcessDataOnMaster(pArgs, iLoop);
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
c9c755c35cd95c006f7ecdb07f2fa8d64686ae3dvboxsync * The worker.
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync supdrvTscDeltaMethod2CollectData(&pArgs->uWorker.M2.Data, &pArgs->uMaster.M2.Data.iCurSeqNo, pArgs->uWorker.M2.fLag);
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync#endif /* GIP_TSC_DELTA_METHOD_2 */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsyncstatic int supdrvTscDeltaVerify(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync,
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync PSUPTSCDELTASYNC2 pOtherSync, bool fIsMaster, int64_t iWorkerTscDelta)
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync /*PSUPGIPCPU pGipCpuWorker = pArgs->pWorker; - unused */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync AssertCompile((RT_ELEMENTS(pArgs->uMaster.Verify.auTscs) & 1) == 0);
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync AssertCompile(RT_ELEMENTS(pArgs->uMaster.Verify.auTscs) == RT_ELEMENTS(pArgs->uWorker.Verify.auTscs));
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * Collect TSC, master goes first.
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync for (i = 0; i < RT_ELEMENTS(pArgs->uMaster.Verify.auTscs); i += 2)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /* Read, kick & wait #1. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO_GO);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /* Read, kick & wait #2. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO_GO)
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * Process the data.
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync for (i = 0; i < RT_ELEMENTS(pArgs->uMaster.Verify.auTscs); i++)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /* Master vs previous worker entry. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync uint64_t uTscMaster = pArgs->uMaster.Verify.auTscs[i] - pGipCpuMaster->i64TSCDelta;
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteS32(&pArgs->rcVerify, VERR_OUT_OF_RANGE);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /* Worker vs master. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync uTscWorker = pArgs->uWorker.Verify.auTscs[i] - iWorkerTscDelta;
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteS32(&pArgs->rcVerify, VERR_OUT_OF_RANGE);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /* Done. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * The worker, master leads.
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync for (i = 0; i < RT_ELEMENTS(pArgs->uWorker.Verify.auTscs); i += 2)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /* Wait, Read and Kick #1. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO_GO);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync /* Wait, Read and Kick #2. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO_GO)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO);
22362edec993ca6ca17ae026d28a8495fad880e0vboxsync TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * Timed out, please retry.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteS32(&pArgs->rcVerify, VERR_TRY_AGAIN);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * Handles the special abort procedure during synchronization setup in
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * supdrvMeasureTscDeltaCallbackUnwrapped().
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * @returns 0 (dummy, ignored)
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * @param pArgs Pointer to argument/state data.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * @param pMySync Pointer to my sync structure.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * @param fIsMaster Set if we're the master, clear if worker.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * @param fTimeout Set if it's a timeout.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsyncsupdrvMeasureTscDeltaCallbackAbortSyncSetup(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, bool fIsMaster, bool fTimeout)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync PSUPTSCDELTASYNC2 volatile *ppMySync = fIsMaster ? &pArgs->pSyncMaster : &pArgs->pSyncWorker;
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync PSUPTSCDELTASYNC2 volatile *ppOtherSync = fIsMaster ? &pArgs->pSyncWorker : &pArgs->pSyncMaster;
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * Clear our sync pointer and make sure the abort flag is set.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * Make sure the other party is out of there and won't be touching our
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * sync state again (would cause stack corruption).
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync while (ASMAtomicReadPtrT(ppOtherSync, PSUPTSCDELTASYNC2) != NULL)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * This is used by supdrvMeasureInitialTscDeltas() to read the TSC on two CPUs
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * and compute the delta between them.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * To reduce code size a good when timeout handling was added, a dummy return
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * value had to be added (saves 1-3 lines per timeout case), thus this
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * 'Unwrapped' function and the dummy 0 return value.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * @returns 0 (dummy, ignored)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param idCpu The CPU we are current scheduled on.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * @param pArgs Pointer to a parameter package.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @remarks Measuring TSC deltas between the CPUs is tricky because we need to
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * read the TSC at exactly the same time on both the master and the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * worker CPUs. Due to DMA, bus arbitration, cache locality,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * contention, SMI, pipelining etc. there is no guaranteed way of
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * doing this on x86 CPUs.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsyncstatic int supdrvMeasureTscDeltaCallbackUnwrapped(RTCPUID idCpu, PSUPDRVGIPTSCDELTARGS pArgs)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync bool const fIsMaster = idCpu == pGipCpuMaster->idCpu;
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync PSUPTSCDELTASYNC2 volatile *ppMySync = fIsMaster ? &pArgs->pSyncMaster : &pArgs->pSyncWorker;
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync PSUPTSCDELTASYNC2 volatile *ppOtherSync = fIsMaster ? &pArgs->pSyncWorker : &pArgs->pSyncMaster;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /* A bit of paranoia first. */
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * If the CPU isn't part of the measurement, return immediately.
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync * Set up my synchronization stuff and wait for the other party to show up.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * We don't wait forever since the other party may be off fishing (offline,
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * spinning with ints disables, whatever), we must play nice to the rest of
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * the system as this context generally isn't one in which we will get
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * preempted and we may hold up a number of lower priority interrupts.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync ASMAtomicWriteU32(&MySync.uSyncVar, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT);
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync /* Look for the partner, might not be here yet... Special abort considerations. */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync while ((pOtherSync = ASMAtomicReadPtrT(ppOtherSync, PSUPTSCDELTASYNC2)) == NULL)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync || !RTMpIsCpuOnline(fIsMaster ? pGipCpuWorker->idCpu : pGipCpuWorker->idCpu) )
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /*fTimeout*/);
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync && ASMReadTSC() - MySync.uTscStart > pArgs->cMaxTscTicks)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, true /*fTimeout*/);
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync /* I found my partner, waiting to be found... Special abort considerations. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* parnaoia */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /*fTimeout*/);
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync while (ASMAtomicReadU32(&MySync.uSyncVar) == GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /*fTimeout*/);
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync && ASMReadTSC() - MySync.uTscStart > pArgs->cMaxTscTicks)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync && !ASMAtomicCmpXchgU32(&MySync.uSyncVar, GIP_TSC_DELTA_SYNC2_PRESTART_ABORT, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT))
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync break; /* race #1: slave has moved on, handle timeout in loop instead. */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, true /*fTimeout*/);
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* race #1 */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /*fTimeout*/);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync/** @todo Add a resumable state to pArgs so we don't waste time if we time
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * out or something. Timeouts are legit, any of the two CPUs may get
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * interrupted. */
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * Start by seeing if we have a zero delta between the two CPUs.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * This should normally be the case.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync rc = supdrvTscDeltaVerify(pArgs, &MySync, pOtherSync, fIsMaster, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * If the verification didn't time out, do regular delta measurements.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * We retry this until we get a reasonable value.
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync * Check the state before we start.
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync uint32_t u32Tmp = ASMAtomicReadU32(&MySync.uSyncVar);
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync && (fIsMaster || u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/loop/%s: #0 iTry=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iTry, u32Tmp));
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync * Do the measurements.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync supdrvTscDeltaMethod1Loop(pArgs, &MySync, pOtherSync, fIsMaster, iTry);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync supdrvTscDeltaMethod2Loop(pArgs, &MySync, pOtherSync, fIsMaster, iTry);
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync * Check the state.
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync && (fIsMaster || u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG(("sync/loop/master: #1 iTry=%u MyState=%#x\n", iTry, u32Tmp));
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG2(("sync/loop/worker: #1 iTry=%u MyState=%#x\n", iTry, u32Tmp));
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync * Success? If so, stop trying. Master decides.
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
ea0f9836d9efcee2e0cabf403a61d6310939c310vboxsync RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
6e1cd642b67bf605db524a9919e28138aeaedba9vboxsync TSCDELTA_DBG_SYNC_MSG2(("sync/loop/master: #9 iTry=%u MyState=%#x\n", iTry, MySync.uSyncVar));
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * End the synchroniziation dance. We tell the other that we're done,
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * then wait for the same kind of reply.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_FINAL);
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync while (ASMAtomicReadU32(&MySync.uSyncVar) != GIP_TSC_DELTA_SYNC2_FINAL)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync && !RTMpIsCpuOnline(fIsMaster ? pGipCpuWorker->idCpu : pGipCpuWorker->idCpu))
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync break; /* this really shouldn't happen. */
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync * Collect some runtime stats.
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync pArgs->cElapsedMasterTscTicks = ASMReadTSC() - MySync.uTscStart;
fcaae7b928d7e61427e8b624046a785507e5a3b6vboxsync pArgs->cElapsedWorkerTscTicks = ASMReadTSC() - MySync.uTscStart;
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * Callback used by supdrvMeasureInitialTscDeltas() to read the TSC on two CPUs
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * and compute the delta between them.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * @param idCpu The CPU we are current scheduled on.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * @param pvUser1 Pointer to a parameter package (SUPDRVGIPTSCDELTARGS).
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync * @param pvUser2 Unused.
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsyncstatic DECLCALLBACK(void) supdrvMeasureTscDeltaCallback(RTCPUID idCpu, void *pvUser1, void *pvUser2)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync supdrvMeasureTscDeltaCallbackUnwrapped(idCpu, (PSUPDRVGIPTSCDELTARGS)pvUser1);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Measures the TSC delta between the master GIP CPU and one specified worker
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns VBox status code.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED on pure measurement
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pDevExt Pointer to the device instance data.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param idxWorker The index of the worker CPU from the GIP's array of
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @remarks This must be called with preemption enabled!
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncstatic int supdrvMeasureTscDeltaOne(PSUPDRVDEVEXT pDevExt, uint32_t idxWorker)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[idxWorker];
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /* Validate input a bit. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Don't attempt measuring the delta for the GIP master.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (pGipCpuWorker->i64TSCDelta == INT64_MAX) /* This shouldn't happen, but just in case. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * One measurement at at time, at least for now. We might be using
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * broadcast IPIs so, so be nice to the rest of the system.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync rc = RTSemMutexRequest(pDevExt->mtxTscDelta, RT_INDEFINITE_WAIT);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * If the CPU has hyper-threading and the APIC IDs of the master and worker are adjacent,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * try pick a different master. (This fudge only works with multi core systems.)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * ASSUMES related threads have adjacent APIC IDs. ASSUMES two threads per core.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * We skip this on AMDs for now as their HTT is different from intel's and
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * it doesn't seem to have any favorable effect on the results.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * If the master is offline, we need a new master too, so share the code.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync iGipCpuMaster = supdrvGipFindCpuIndexForCpuId(pGip, idMaster);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertReturn(iGipCpuMaster < pGip->cCpus, VERR_INVALID_CPU_ID);
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if ( ( (pGipCpuMaster->idApic & ~1) == (pGipCpuWorker->idApic & ~1)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync && pGip->aCPUs[i].enmState == SUPGIPCPUSTATE_ONLINE
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync && pGip->aCPUs[i].idCpu != idMaster /* paranoia starts here... */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpuWorker->iCpuSet))
e413b055bbccf1f99a47bcf7ef49846ca02781a1vboxsync * Initialize data package for the RTMpOnPair callback.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync PSUPDRVGIPTSCDELTARGS pArgs = (PSUPDRVGIPTSCDELTARGS)RTMemAllocZ(sizeof(*pArgs));
b0deb52d71d46987430324c2e39de46a2791a738vboxsync pArgs->cMaxTscTicks = ASMAtomicReadU64(&pGip->u64CpuHz) / 512; /* 1953 us */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync * Do the RTMpOnPair call. We reset i64TSCDelta first so we
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync * and supdrvMeasureTscDeltaCallback can use it as a success check.
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** @todo Store the i64TSCDelta result in pArgs first? Perhaps deals with
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync * that when doing the restart loop reorg. */
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, INT64_MAX);
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync rc = RTMpOnPair(pGipCpuMaster->idCpu, pGipCpuWorker->idCpu, RTMPON_F_CONCURRENT_EXEC,
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync SUPR0Printf("mponpair ticks: %9llu %9llu max: %9llu iTry: %u%s\n", pArgs->cElapsedMasterTscTicks,
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync pArgs->cElapsedWorkerTscTicks, pArgs->cMaxTscTicks, pArgs->iTry,
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync SUPR0Printf("rcVerify=%d iVerifyBadTscDiff=%lld cMinVerifyTscTicks=%lld cMaxVerifyTscTicks=%lld\n",
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync pArgs->rcVerify, pArgs->iVerifyBadTscDiff, pArgs->cMinVerifyTscTicks, pArgs->cMaxVerifyTscTicks);
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync if (RT_LIKELY(pGipCpuWorker->i64TSCDelta != INT64_MAX))
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync * Work the TSC delta applicability rating. It starts
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync * optimistic in supdrvGipInit, we downgrade it here.
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync || pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO)
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync else if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync || pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO)
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync AssertCompile(sizeof(pGip->enmUseTscDelta) == sizeof(uint32_t));
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync ASMAtomicWriteU32((uint32_t volatile *)&pGip->enmUseTscDelta, enmRating);
ccf651c487e8410e36b2bd79a406a202c02c417fvboxsync /** @todo return try-again if we get an offline CPU error. */
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync * We're done now.
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync rc2 = RTSemMutexRelease(pDevExt->mtxTscDelta); AssertRC(rc2);
d5dbc72216583ef566ba070ac5613e0bca155f82vboxsync rc2 = RTSemFastMutexRelease(pDevExt->mtxTscDelta); AssertRC(rc2);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * Resets the TSC-delta related TSC samples and optionally the deltas
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * themselves.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @param pDevExt Pointer to the device instance data.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @param fResetTscDeltas Whether the TSC-deltas are also to be reset.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @remarks This might be called while holding a spinlock!
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsyncstatic void supdrvTscResetSamples(PSUPDRVDEVEXT pDevExt, bool fResetTscDeltas)
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync ASMAtomicWriteU64(&pGipCpu->u64TSCSample, GIP_TSC_DELTA_RSVD);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync RTCpuSetDelByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpu->iCpuSet);
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync ASMAtomicWriteS64(&pGipCpu->i64TSCDelta, INT64_MAX);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * Picks an online CPU as the master TSC for TSC-delta computations.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns VBox status code.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @param pDevExt Pointer to the device instance data.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @param pidxMaster Where to store the CPU array index of the chosen
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * master. Optional, can be NULL.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsyncstatic int supdrvTscPickMaster(PSUPDRVDEVEXT pDevExt, uint32_t *pidxMaster)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Pick the first CPU online as the master TSC and make it the new GIP master based
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * on the APIC ID.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Technically we can simply use "idGipMaster" but doing this gives us master as CPU 0
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * in most cases making it nicer/easier for comparisons. It is safe to update the GIP
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * master as this point since the sync/async timer isn't created yet.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync for (iCpu = 0; iCpu < RT_ELEMENTS(pGip->aiCpuFromApicId); iCpu++)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpu->iCpuSet))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pGipCpu->i64TSCDelta = GIP_TSC_DELTA_INITIAL_MASTER_VALUE;
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync ASMAtomicWriteSize(&pDevExt->idGipMaster, pGipCpu->idCpu);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * Performs the initial measurements of the TSC deltas between CPUs.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * This is called by supdrvGipCreate(), supdrvGipPowerNotificationCallback() or
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * triggered by it if threaded.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @returns VBox status code.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @param pDevExt Pointer to the device instance data.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @remarks Must be called only after supdrvGipInitOnCpu() as this function uses
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * idCpu, GIP's online CPU set which are populated in
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * supdrvGipInitOnCpu().
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsyncstatic int supdrvMeasureInitialTscDeltas(PSUPDRVDEVEXT pDevExt)
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync uint32_t cMpOnOffEvents = ASMAtomicReadU32(&pDevExt->cMpOnOffEvents);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync supdrvTscResetSamples(pDevExt, true /* fClearDeltas */);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync SUPR0Printf("Failed to pick a CPU master for TSC-delta measurements rc=%Rrc\n", rc);
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync AssertReturn(idxMaster < pGip->cCpus, VERR_INVALID_CPU_INDEX);
d5d73d91f3aba093b9737fc51ccf3c7a31501d80vboxsync Assert(pDevExt->idGipMaster == pGipCpuMaster->idCpu);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * If there is only a single CPU online we have nothing to do.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertReturn(pGip->cOnlineCpus > 0, VERR_INTERNAL_ERROR_5);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Loop thru the GIP CPU array and get deltas for each CPU (except the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * master). We do the CPUs with the even numbered APIC IDs first so that
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * we've got alternative master CPUs to pick from on hyper-threaded systems.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync && (iOddEven > 0 || (pGipCpuWorker->idApic & 1) == 0)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync && RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync SUPR0Printf("supdrvMeasureTscDeltaOne failed. rc=%d CPU[%u].idCpu=%u Master[%u].idCpu=%u\n", rc, iCpu,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync pGipCpuWorker->idCpu, idxMaster, pDevExt->idGipMaster, pGipCpuMaster->idCpu);
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (ASMAtomicReadU32(&pDevExt->cMpOnOffEvents) != cMpOnOffEvents)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync SUPR0Printf("One or more CPUs transitioned between online & offline states. I'm confused, retry...\n");
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Switches the TSC-delta measurement thread into the butchered state.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns VBox status code.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt Pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param fSpinlockHeld Whether the TSC-delta spinlock is held or not.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pszFailed An error message to log.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param rcFailed The error code to exit the thread with.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic int supdrvTscDeltaThreadButchered(PSUPDRVDEVEXT pDevExt, bool fSpinlockHeld, const char *pszFailed, int rcFailed)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Butchered;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaThreadButchered: %s. rc=%Rrc\n", rcFailed));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * The TSC-delta measurement thread.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns VBox status code.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param hThread The thread handle.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pvUser Opaque pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic DECLCALLBACK(int) supdrvTscDeltaThread(RTTHREAD hThread, void *pvUser)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Switch on the current state.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* fall thru */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Simple adaptive timeout. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync else if (pDevExt->cMsTscDeltaTimeout == 10) /* +100 ms */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync else if (pDevExt->cMsTscDeltaTimeout == 100) /* +1000 ms */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTThreadUserWait(pDevExt->hTscDeltaThread, pDevExt->cMsTscDeltaTimeout);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync return supdrvTscDeltaThreadButchered(pDevExt, false /* fSpinlockHeld */, "RTThreadUserWait", rc);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Measuring;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTSemEventSignal(pDevExt->hTscDeltaEvent); /* (Safe on windows as long as spinlock isn't IRQ safe.) */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* fall thru */
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync RTCpuSetCopy(&pDevExt->TscDeltaCpuSet, &pGip->OnlineCpuSet);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync } while (cTries-- > 0);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Measure TSC-deltas only for the CPUs that are in the set. */
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync if (RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * The thread/someone must've called SUPR0TscDeltaMeasureBySetIndex(),
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync * mark the delta as fine to get the timer thread off our back.
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
ddcb64df779903f563ad28b5b018bcf94ba22a56vboxsync RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync Assert(rc != VERR_NOT_AVAILABLE); /* VERR_NOT_AVAILABLE is used as init value, see supdrvTscDeltaThreadInit(). */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Destroyed;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "Invalid state", VERR_INVALID_STATE);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Waits for the TSC-delta measurement thread to respond to a state change.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns VINF_SUCCESS on success, VERR_TIMEOUT if it doesn't respond in time,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * other error code on internal error.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pThis Pointer to the grant service instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param enmCurState The current state.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param enmNewState The new state we're waiting for it to enter.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic int supdrvTscDeltaThreadWait(PSUPDRVDEVEXT pDevExt, SUPDRVTSCDELTATHREADSTATE enmCurState,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Wait a short while for the expected state transition.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync RTSemEventWait(pDevExt->hTscDeltaEvent, RT_MS_1SEC);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync if (pDevExt->enmTscDeltaThreadState == enmNewState)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync else if (pDevExt->enmTscDeltaThreadState == enmCurState)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Wait longer if the state has not yet transitioned to the one we want.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTSemEventWait(pDevExt->hTscDeltaEvent, 50 * RT_MS_1SEC);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Check the state whether we've succeeded.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaThreadWait: timed out state transition. enmState=%d enmNewState=%d\n", enmState,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d, expected %d\n", enmCurState,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaThreadWait: RTSemEventWait failed. rc=%Rrc\n", rc));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d\n", enmCurState, enmNewState));
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * Signals the TSC-delta thread to start measuring TSC-deltas.
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * @param pDevExt Pointer to the device instance data.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync * @param fForceAll Force re-calculating TSC-deltas on all CPUs.
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsyncstatic void supdrvTscDeltaThreadStartMeasurement(PSUPDRVDEVEXT pDevExt, bool fForceAll)
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync || pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
b07cf09ce59f674b9c085bcd7ae77927cd551804vboxsync else if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_WaitAndMeasure
ece539e847eaf73e9a1910b562d998dc0afc5580vboxsync * Terminates the actual thread running supdrvTscDeltaThread().
ece539e847eaf73e9a1910b562d998dc0afc5580vboxsync * This is an internal worker function for supdrvTscDeltaThreadInit() and
ece539e847eaf73e9a1910b562d998dc0afc5580vboxsync * supdrvTscDeltaTerm().
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt Pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvTscDeltaThreadTerminate(PSUPDRVDEVEXT pDevExt)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Terminating;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTThreadWait(pDevExt->hTscDeltaThread, 50 * RT_MS_1SEC, NULL /* prc */);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync /* Signal a few more times before giving up. */
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync while (--cTriesLeft > 0)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTThreadWait(pDevExt->hTscDeltaThread, 2 * RT_MS_1SEC, NULL /* prc */);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Initializes and spawns the TSC-delta measurement thread.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * A thread is required for servicing re-measurement requests from events like
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * CPUs coming online, suspend/resume etc. as it cannot be done synchronously
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * under all contexts on all OSs.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @returns VBox status code.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt Pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @remarks Must only be called -after- initializing GIP and setting up MP
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * notifications!
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync Assert(pDevExt->pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTSpinlockCreate(&pDevExt->hTscDeltaSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "VBoxTscSpnLck");
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Creating;
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = RTThreadCreate(&pDevExt->hTscDeltaThread, supdrvTscDeltaThread, pDevExt, 0 /* cbStack */,
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync RTTHREADTYPE_DEFAULT, RTTHREADFLAGS_WAITABLE, "VBoxTscThread");
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync rc = supdrvTscDeltaThreadWait(pDevExt, kTscDeltaThreadState_Creating, kTscDeltaThreadState_Listening);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaInit: supdrvTscDeltaThreadWait failed. rc=%Rrc\n", rc));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaInit: RTThreadCreate failed. rc=%Rrc\n", rc));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaInit: RTSemEventCreate failed. rc=%Rrc\n", rc));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync OSDBGPRINT(("supdrvTscDeltaInit: RTSpinlockCreate failed. rc=%Rrc\n", rc));
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * Terminates the TSC-delta measurement thread and cleanup.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync * @param pDevExt Pointer to the device instance data.
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsyncstatic void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt)
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
50671c30431539dd7d6ff6a5f2ceb6c9f9f471b2vboxsync#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * Measure the TSC delta for the CPU given by its CPU set index.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns VBox status code.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @retval VERR_INTERRUPTED if interrupted while waiting.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED if we were unable to get a
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * measurment.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @retval VERR_CPU_OFFLINE if the specified CPU is offline.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param pSession The caller's session. GIP must've been mapped.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param iCpuSet The CPU set index of the CPU to measure.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param fFlags Flags, SUP_TSCDELTA_MEASURE_F_XXX.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param cMsWaitRetry Number of milliseconds to wait between each retry.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param cMsWaitThread Number of milliseconds to wait for the thread to get
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param cTries Number of times to try, pass 0 for the default.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsyncSUPR0DECL(int) SUPR0TscDeltaMeasureBySetIndex(PSUPDRVSESSION pSession, uint32_t iCpuSet, uint32_t fFlags,
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync RTMSINTERVAL cMsWaitRetry, RTMSINTERVAL cMsWaitThread, uint32_t cTries)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * Validate and adjust the input.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync AssertReturn(SUP_IS_DEVEXT_VALID(pDevExt), VERR_INVALID_PARAMETER);
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync AssertReturn(iCpuSet < RTCPUSET_MAX_CPUS, VERR_INVALID_CPU_INDEX);
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync AssertReturn(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx), VERR_INVALID_CPU_INDEX);
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync AssertReturn(iGipCpu < pGip->cCpus, VERR_INVALID_CPU_INDEX);
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync * The request is a noop if the TSC delta isn't being used.
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync if (pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * Has the TSC already been measured and we're not forced to redo it?
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * Asynchronous request? Forward it to the thread, no waiting.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync /** @todo Async. doesn't implement options like retries, waiting. We'll need
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * to pass those options to the thread somehow and implement it in the
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * thread. Check if anyone uses/needs fAsync before implementing this. */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, iCpuSet);
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync || pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync else if (pDevExt->enmTscDeltaThreadState != kTscDeltaThreadState_WaitAndMeasure)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * If a TSC-delta measurement request is already being serviced by the thread,
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * wait 'cTries' times if a retry-timeout is provided, otherwise bail as busy.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync { /* Must wait, the thread is busy. */ }
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync else if (enmState == kTscDeltaThreadState_WaitAndMeasure)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync { /* Must wait, this state only says what will happen next. */ }
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync else if (enmState == kTscDeltaThreadState_Terminating)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync { /* Must wait, this state only says what should happen next. */ }
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync break; /* All other states, the thread is either idly listening or dead. */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync /* Wait or fail. */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync cMsElapsed = RTTimeSystemMilliTS() - msTsStartWait;
87c5113417e917cdf64545d4f8e0a27047cea783vboxsync rc = RTThreadSleep(RT_MIN((RTMSINTERVAL)(cMsWaitThread - cMsElapsed), RT_MIN(iWaitLoop + 1, 10)));
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * Try measure the TSC delta the given number of times.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync /* Unless we're forced to measure the delta, check whether it's done already. */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync /* Measure it. */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync if (rc != VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync Assert(pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX || RT_FAILURE_NP(rc));
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync /* Retry? */
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync /* Always delay between retries (be nice to the rest of the system
6f34d825e07e8dcd535b37d7685ab62220a23addvboxsync and avoid the BSOD hounds). */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * Service a TSC-delta measurement request.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @returns VBox status code.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param pDevExt Pointer to the device instance data.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param pSession The support driver session.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * @param pReq Pointer to the TSC-delta measurement request.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsyncint VBOXCALL supdrvIOCtl_TscDeltaMeasure(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCDELTAMEASURE pReq)
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync * Validate and adjust/resolve the input so they can be passed onto SUPR0TscDeltaMeasureBySetIndex.
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync AssertPtr(pDevExt); AssertPtr(pSession); AssertPtr(pReq); /* paranoia^2 */
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync cTries = pReq->u.In.cRetries == 0 ? 0 : (uint32_t)pReq->u.In.cRetries + 1;
9a59b271202449cf51ec3ac13b8ef1b4b9810850vboxsync return SUPR0TscDeltaMeasureBySetIndex(pSession, iCpuSet, fFlags, cMsWaitRetry,
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync cTries == 0 ? 5 * RT_MS_1SEC : cMsWaitRetry * cTries /*cMsWaitThread*/,
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Reads TSC with delta applied.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Will try to resolve delta value INT64_MAX before applying it. This is the
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * main purpose of this function, to handle the case where the delta needs to be
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * determined.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @returns VBox status code.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pDevExt Pointer to the device instance data.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pSession The support driver session.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * @param pReq Pointer to the TSC-read request.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsyncint VBOXCALL supdrvIOCtl_TscRead(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCREAD pReq)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Validate. We require the client to have mapped GIP (no asserting on
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * ring-3 preconditions).
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertPtr(pDevExt); AssertPtr(pReq); AssertPtr(pSession); /* paranoia^2 */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * We're usually here because we need to apply delta, but we shouldn't be
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * upset if the GIP is some different mode.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * Start by gathering the data, using CLI for disabling preemption
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * while we do that.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync && (iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]) < pGip->cCpus ))
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync int64_t i64Delta = pGip->aCPUs[iGipCpu].i64TSCDelta;
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * If we're lucky we've got a delta, but no predicitions here
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * as this I/O control is normally only used when the TSC delta
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * is set to INT64_MAX.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /* Give up after a few times. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /* Need to measure the delta an try again. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync Assert(pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX || RT_FAILURE_NP(rc));
476493afbafe452ee52b3b3b2bb77e07e5e56285vboxsync /** @todo should probably delay on failure... dpc watchdogs */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync /* This really shouldn't happen. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync AssertMsgFailed(("idCpu=%#x iCpuSet=%#x (%d)\n", RTMpCpuId(), iCpuSet, iCpuSet));
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync rc = VERR_INTERNAL_ERROR_5; /** @todo change to warning. */
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync * No delta to apply. Easy. Deal with preemption the lazy way.
9888fffcfbe2d41dce14a1249b12cb88cc9b149fvboxsync if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)