/* $Id$ */
/** @file
* IPRT - Multiprocessor, Ring-0 Driver, NT.
*/
/*
* Copyright (C) 2008-2014 Oracle Corporation
*
* This file is part of VirtualBox Open Source Edition (OSE), as
* available from http://www.virtualbox.org. This file is free software;
* General Public License (GPL) as published by the Free Software
* Foundation, in version 2 as it comes in the "COPYING" file of the
* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
*
* The contents of this file may alternatively be used under the terms
* of the Common Development and Distribution License Version 1.0
* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
* VirtualBox OSE distribution, in which case the provisions of the
* CDDL are applicable instead of those of the GPL.
*
* You may elect to license modified versions of this file under the
* terms and conditions of either the GPL or the CDDL or both.
*/
/*******************************************************************************
* Header Files *
*******************************************************************************/
#include "the-nt-kernel.h"
#include "r0drv/mp-r0drv.h"
#include "internal-r0drv-nt.h"
/*******************************************************************************
* Structures and Typedefs *
*******************************************************************************/
typedef enum
{
} RT_NT_CPUID;
/**
* Used by the RTMpOnSpecific.
*/
typedef struct RTMPNTONSPECIFICARGS
{
/** Set if we're executing. */
bool volatile fExecuting;
/** Set when done executing. */
bool volatile fDone;
/** Number of references to this heap block. */
/** Event that the calling thread is waiting on. */
/** The deferred procedure call object. */
/** The callback argument package. */
/* test a couple of assumption. */
/** @todo
* We cannot do other than assume a 1:1 relationship between the
* the docs. If someone knows a better way to get this done, please
* let bird know.
*/
{
/* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */
return KeGetCurrentProcessorNumber();
}
{
/* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */
return KeGetCurrentProcessorNumber();
}
{
return *pidCpu = KeGetCurrentProcessorNumber();
}
{
}
{
}
{
/** @todo use KeQueryMaximumProcessorCount on vista+ */
return MAXIMUM_PROCESSORS - 1;
}
{
if (idCpu >= MAXIMUM_PROCESSORS)
return false;
#if 0 /* this isn't safe at all IRQLs (great work guys) */
#else
#endif
}
{
/* Cannot easily distinguish between online and offline cpus. */
* (KeQueryMaximumProcessorCount). */
return RTMpIsCpuOnline(idCpu);
}
{
* (KeQueryMaximumProcessorCount). */
return RTMpGetOnlineSet(pSet);
}
{
* (KeQueryMaximumProcessorCount). */
return RTMpGetOnlineCount();
}
{
#if 0 /* this isn't safe at all IRQLs (great work guys) */
#else
*pSet = g_rtMpNtCpuSet;
return pSet;
#endif
}
{
return RTCpuSetCount(&Set);
}
#if 0
/* Experiment with checking the undocumented KPRCB structure
* 'dt nt!_kprcb 0xaddress' shows the layout
*/
typedef struct
{
volatile ULONG DpcQueueDepth;
} KDPC_DATA, *PKDPC_DATA;
RTDECL(bool) RTMpIsCpuWorkPending(void)
{
_asm {
}
if (pDpcData->DpcQueueDepth)
return true;
pDpcData++;
if (pDpcData->DpcQueueDepth)
return true;
return false;
}
#else
{
/** @todo not implemented */
return false;
}
#endif
/**
* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
* the RTMpOnAll case.
*
* @param uUserCtx The user context argument (PRTMPARGS).
*/
{
/*ASMAtomicIncU32(&pArgs->cHits); - not needed */
return 0;
}
/**
* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
* the RTMpOnOthers case.
*
* @param uUserCtx The user context argument (PRTMPARGS).
*/
{
{
/*ASMAtomicIncU32(&pArgs->cHits); - not needed */
}
return 0;
}
/**
* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
* the RTMpOnPair case.
*
* @param uUserCtx The user context argument (PRTMPARGS).
*/
{
{
}
return 0;
}
/**
* Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
* the RTMpOnSpecific case.
*
* @param uUserCtx The user context argument (PRTMPARGS).
*/
{
{
}
return 0;
}
/**
* Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
*
* @returns VINF_SUCCESS.
* @param pfnWorker The callback.
* @param pvUser1 User argument 1.
* @param pvUser2 User argument 2.
* @param idCpu First CPU to match, ultimately specific to the
* pfnNativeWrapper used.
* @param idCpu2 Second CPU to match, ultimately specific to the
* pfnNativeWrapper used.
* @param pcHits Where to return the number of this. Optional.
*/
{
if (pcHits)
return VINF_SUCCESS;
}
/**
* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
*
* @param Dpc DPC object
* @param DeferredContext Context argument specified by KeInitializeDpc
* @param SystemArgument1 Argument specified by KeInsertQueueDpc
* @param SystemArgument2 Argument specified by KeInsertQueueDpc
*/
static VOID __stdcall rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
{
/* Dereference the argument structure. */
if (cRefs == 0)
}
/**
* Internal worker for the RTMpOn* APIs.
*
* @returns IPRT status code.
* @param pfnWorker The callback.
* @param pvUser1 User argument 1.
* @param pvUser2 User argument 2.
* @param enmCpuid What to do / is idCpu valid.
* @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
* RT_NT_CPUID_PAIR, otherwise ignored.
* @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
* @param pcHits Where to return the number of this. Optional.
*/
{
#if 0
/* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
* driver verifier doesn't complain...
*/
AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
#endif
#ifdef IPRT_TARGET_NT4
/* g_pfnrtNt* are not present on NT anyway. */
return VERR_NOT_SUPPORTED;
#else
#endif
/* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
return VERR_NOT_SUPPORTED;
pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, MAXIMUM_PROCESSORS*sizeof(KDPC) + sizeof(RTMPARGS), (ULONG)'RTMp');
if (!pArgs)
return VERR_NO_MEMORY;
if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
}
else if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
}
else
{
for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
{
KeSetTargetProcessorDpc(&paExecCpuDpcs[i], i);
}
}
/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
*/
/*
* We cannot do other than assume a 1:1 relationship between the
* affinity mask and the process despite the warnings in the docs.
* If someone knows a better way to get this done, please let bird know.
*/
ASMCompilerBarrier(); /* paranoia */
if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
}
else if (enmCpuid == RT_NT_CPUID_PAIR)
{
}
else
{
for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
{
if ( (i != iSelf)
{
}
}
if (enmCpuid != RT_NT_CPUID_OTHERS)
}
/* Flush all DPCs and wait for completion. (can take long!) */
* stuff (and check for the current cpu above in the specific case). */
/** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
* executed. Seen pArgs being freed while some CPU was using it before
* cRefs was added. */
if (pcHits)
/* Dereference the argument structure. */
if (cRefs == 0)
return VINF_SUCCESS;
}
{
return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
}
{
return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
}
RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
int rc;
return VERR_NOT_SUPPORTED;
/*
* Check that both CPUs are online before doing the broadcast call.
*/
if ( RTMpIsCpuOnline(idCpu1)
&& RTMpIsCpuOnline(idCpu2))
{
/*
* The broadcast IPI isn't quite as bad as it could have been, because
* it looks like windows doesn't synchronize CPUs on the way out, they
* seems to get back to normal work while the pair is still busy.
*/
rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
else
if (RT_SUCCESS(rc))
{
if (cHits == 2)
rc = VINF_SUCCESS;
else if (cHits == 1)
else if (cHits == 0)
else
rc = VERR_CPU_IPE_1;
}
}
/*
* A CPU must be present to be considered just offline.
*/
else if ( RTMpIsCpuPresent(idCpu1)
&& RTMpIsCpuPresent(idCpu2))
else
return rc;
}
{
return g_pfnrtKeIpiGenericCall != NULL;
}
/**
* Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
* when the last reference is released.
*/
{
if (cRefs == 0)
}
/**
* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
*
* @param Dpc DPC object
* @param DeferredContext Context argument specified by KeInitializeDpc
* @param SystemArgument1 Argument specified by KeInsertQueueDpc
* @param SystemArgument2 Argument specified by KeInsertQueueDpc
*/
{
pArgs->CallbackArgs.pfnWorker(KeGetCurrentProcessorNumber(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
}
{
/*
* Don't try mess with an offline CPU.
*/
if (!RTMpIsCpuOnline(idCpu))
return !RTMpIsCpuPossible(idCpu)
/*
* Use the broadcast IPI routine if there are no more than two CPUs online,
* or if the current IRQL is unsuitable for KeWaitForSingleObject.
*/
int rc;
&& ( RTMpGetOnlineCount() <= 2
|| KeGetCurrentIrql() > APC_LEVEL)
)
{
if (RT_SUCCESS(rc))
{
if (cHits == 1)
return VINF_SUCCESS;
}
return rc;
}
#if 0
rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
if (RT_SUCCESS(rc))
{
if (cHits == 1)
return VINF_SUCCESS;
}
return rc;
#else
/*
* Initialize the argument package and the objects within it.
* The package is referenced counted to avoid unnecessary spinning to
* synchronize cleanup and prevent stack corruption.
*/
PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)ExAllocatePoolWithTag(NonPagedPool, sizeof(*pArgs), (ULONG)'RTMp');
if (!pArgs)
return VERR_NO_MEMORY;
pArgs->fExecuting = false;
/*
* Disable preemption while we check the current processor and inserts the DPC.
*/
ASMCompilerBarrier(); /* paranoia */
{
/* Just execute the callback on the current CPU. */
return VINF_SUCCESS;
}
/* Different CPU, so queue it if the CPU is still online. */
if (RTMpIsCpuOnline(idCpu))
{
/*
*/
while (cLoopsLeft-- > 0)
{
{
return VINF_SUCCESS;
}
ASMNopPause();
}
/*
* It didn't respond, so wait on the event object, poking the CPU if it's slow.
*/
NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
if (rcNt == STATUS_SUCCESS)
{
return VINF_SUCCESS;
}
/* If it hasn't respondend yet, maybe poke it and wait some more. */
if (rcNt == STATUS_TIMEOUT)
{
#ifndef IPRT_TARGET_NT4
if ( !pArgs->fExecuting
#endif
rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
if (rcNt == STATUS_SUCCESS)
{
return VINF_SUCCESS;
}
}
/*
* Something weird is happening, try bail out.
*/
{
LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
}
else
{
/* DPC is running, wait a good while for it to complete. */
LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
if (rcNt != STATUS_SUCCESS)
LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
}
}
else
{
/* CPU is offline.*/
}
return rc;
#endif
}
static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
{
}
#ifndef IPRT_TARGET_NT4
/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
{
return 0;
}
/**
* RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
*
* @returns VINF_SUCCESS
* @param idCpu The CPU identifier.
*/
{
return VINF_SUCCESS;
}
/**
* RTMpPokeCpu worker that uses HalSendSoftwareInterrupt to get the job done.
*
* This is only really available on AMD64, at least at the time of writing.
*
* @returns VINF_SUCCESS
* @param idCpu The CPU identifier.
*/
{
return VINF_SUCCESS;
}
/**
* RTMpPokeCpu worker that uses the Windows 7 and later version of
* HalRequestIpip to get the job done.
*
* @returns VINF_SUCCESS
* @param idCpu The CPU identifier.
*/
{
/*
* I think we'll let idCpu be an NT processor number and not a HAL processor
* index. KeAddProcessorAffinityEx is for HAL and uses HAL processor
* indexes as input from what I can tell.
*/
return VINF_SUCCESS;
}
/**
* RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
* to get the job done.
*
* @returns VINF_SUCCESS
* @param idCpu The CPU identifier.
*/
{
__debugbreak(); /** @todo this code needs testing!! */
return VINF_SUCCESS;
}
#endif /* !IPRT_TARGET_NT4 */
{
/*
* APC fallback.
*/
static bool s_fPokeDPCsInitialized = false;
if (!s_fPokeDPCsInitialized)
{
for (unsigned i = 0; i < RT_ELEMENTS(s_aPokeDpcs); i++)
{
KeSetTargetProcessorDpc(&s_aPokeDpcs[i], (int)i);
}
s_fPokeDPCsInitialized = true;
}
/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
*/
/* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
* @note: not true on at least Vista & Windows 7
*/
}
{
if (!RTMpIsCpuOnline(idCpu))
return !RTMpIsCpuPossible(idCpu)
/* Calls rtMpSendIpiFallback, rtMpSendIpiWin7AndLater or rtMpSendIpiVista. */
return g_pfnrtMpPokeCpuWorker(idCpu);
}
{
return false;
}