mp-r0drv-nt.cpp revision 41c15aa44c4acfe6760552150eb931319835cfaf
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync/* $Id$ */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync/** @file
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * IPRT - Multiprocessor, Ring-0 Driver, NT.
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
e64031e20c39650a7bc902a3e1aba613b9415deevboxsync/*
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * Copyright (C) 2008 Sun Microsystems, Inc.
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync *
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * This file is part of VirtualBox Open Source Edition (OSE), as
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * available from http://www.virtualbox.org. This file is free software;
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * you can redistribute it and/or modify it under the terms of the GNU
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * General Public License (GPL) as published by the Free Software
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * Foundation, in version 2 as it comes in the "COPYING" file of the
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync *
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * The contents of this file may alternatively be used under the terms
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * of the Common Development and Distribution License Version 1.0
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * VirtualBox OSE distribution, in which case the provisions of the
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * CDDL are applicable instead of those of the GPL.
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync *
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * You may elect to license modified versions of this file under the
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * terms and conditions of either the GPL or the CDDL or both.
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync *
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * Clara, CA 95054 USA or visit http://www.sun.com if you need
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * additional information or have any questions.
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
590bfe12ce22cd3716448fbb9f4dc51664bfe5e2vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync/*******************************************************************************
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync* Header Files *
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync*******************************************************************************/
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#include "the-nt-kernel.h"
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#include <iprt/mp.h>
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#include <iprt/cpuset.h>
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#include <iprt/err.h>
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#include <iprt/asm.h>
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#include "r0drv/mp-r0drv.h"
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#include "internal-r0drv-nt.h"
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync/*******************************************************************************
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync* Structures and Typedefs *
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync*******************************************************************************/
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsynctypedef enum
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync{
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync RT_NT_CPUID_SPECIFIC,
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync RT_NT_CPUID_OTHERS,
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync RT_NT_CPUID_ALL
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync} RT_NT_CPUID;
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync/* test a couple of assumption. */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsyncAssertCompile(MAXIMUM_PROCESSORS <= RTCPUSET_MAX_CPUS);
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsyncAssertCompile(NIL_RTCPUID >= MAXIMUM_PROCESSORS);
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync/** @todo
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * We cannot do other than assume a 1:1 relationship between the
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * affinity mask and the process despite the vagueness/warnings in
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * the docs. If someone knows a better way to get this done, please
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync * let bird know.
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
ad27e1d5e48ca41245120c331cc88b50464813cevboxsyncRTDECL(RTCPUID) RTMpCpuId(void)
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync{
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync /* WDK upgrade warning: PCR->Number changed from BYTE to WORD. */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return KeGetCurrentProcessorNumber();
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync}
ad27e1d5e48ca41245120c331cc88b50464813cevboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsyncRTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync{
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return idCpu < MAXIMUM_PROCESSORS ? (int)idCpu : -1;
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync}
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsyncRTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync{
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return (unsigned)iCpu < MAXIMUM_PROCESSORS ? iCpu : NIL_RTCPUID;
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync}
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsyncRTDECL(RTCPUID) RTMpGetMaxCpuId(void)
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync{
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync /** @todo use KeQueryMaximumProcessorCount on vista+ */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return MAXIMUM_PROCESSORS - 1;
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync}
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsyncRTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync{
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync if (idCpu >= MAXIMUM_PROCESSORS)
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return false;
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#if 0 /* this isn't safe at all IRQLs (great work guys) */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync KAFFINITY Mask = KeQueryActiveProcessors();
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return !!(Mask & RT_BIT_64(idCpu));
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#else
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync#endif
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync}
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
df5f1b4303be82a000284349e9392beb9f1b6f11vboxsyncRTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
df5f1b4303be82a000284349e9392beb9f1b6f11vboxsync{
df5f1b4303be82a000284349e9392beb9f1b6f11vboxsync /* Cannot easily distinguish between online and offline cpus. */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync /** @todo online/present cpu stuff must be corrected for proper W2K8 support
df5f1b4303be82a000284349e9392beb9f1b6f11vboxsync * (KeQueryMaximumProcessorCount). */
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync return RTMpIsCpuOnline(idCpu);
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync}
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
590bfe12ce22cd3716448fbb9f4dc51664bfe5e2vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsyncRTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
b0ab7e3ff45cca3c9755432dde464562cdde61f0vboxsync{
/** @todo online/present cpu stuff must be corrected for proper W2K8 support
* (KeQueryMaximumProcessorCount). */
return RTMpGetOnlineSet(pSet);
}
RTDECL(RTCPUID) RTMpGetCount(void)
{
/** @todo online/present cpu stuff must be corrected for proper W2K8 support
* (KeQueryMaximumProcessorCount). */
return RTMpGetOnlineCount();
}
RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
{
#if 0 /* this isn't safe at all IRQLs (great work guys) */
KAFFINITY Mask = KeQueryActiveProcessors();
return RTCpuSetFromU64(pSet, Mask);
#else
*pSet = g_rtMpNtCpuSet;
return pSet;
#endif
}
RTDECL(RTCPUID) RTMpGetOnlineCount(void)
{
RTCPUSET Set;
RTMpGetOnlineSet(&Set);
return RTCpuSetCount(&Set);
}
#if 0
/* Experiment with checking the undocumented KPRCB structure
* 'dt nt!_kprcb 0xaddress' shows the layout
*/
typedef struct
{
LIST_ENTRY DpcListHead;
ULONG_PTR DpcLock;
volatile ULONG DpcQueueDepth;
ULONG DpcQueueCount;
} KDPC_DATA, *PKDPC_DATA;
RTDECL(bool) RTMpIsCpuWorkPending(void)
{
uint8_t *pkprcb;
PKDPC_DATA pDpcData;
_asm {
mov eax, fs:0x20
mov pkprcb, eax
}
pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
if (pDpcData->DpcQueueDepth)
return true;
pDpcData++;
if (pDpcData->DpcQueueDepth)
return true;
return false;
}
#else
RTDECL(bool) RTMpIsCpuWorkPending(void)
{
/** @todo not implemented */
return false;
}
#endif
/**
* Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
*
* @param Dpc DPC object
* @param DeferredContext Context argument specified by KeInitializeDpc
* @param SystemArgument1 Argument specified by KeInsertQueueDpc
* @param SystemArgument2 Argument specified by KeInsertQueueDpc
*/
static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
{
PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
ASMAtomicIncU32(&pArgs->cHits);
pArgs->pfnWorker(KeGetCurrentProcessorNumber(), pArgs->pvUser1, pArgs->pvUser2);
}
/**
* Internal worker for the RTMpOn* APIs.
*
* @returns IPRT status code.
* @param pfnWorker The callback.
* @param pvUser1 User argument 1.
* @param pvUser2 User argument 2.
* @param enmCpuid What to do / is idCpu valid.
* @param idCpu Used if enmCpuid RT_NT_CPUID_SPECIFIC, otherwise ignored.
*/
static int rtMpCall(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2, RT_NT_CPUID enmCpuid, RTCPUID idCpu)
{
PRTMPARGS pArgs;
KDPC *paExecCpuDpcs;
#if 0
/* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
* driver verifier doesn't complain...
*/
AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
#endif
#ifdef IPRT_TARGET_NT4
KAFFINITY Mask;
/* g_pfnrtNt* are not present on NT anyway. */
return VERR_NOT_SUPPORTED;
#else
KAFFINITY Mask = KeQueryActiveProcessors();
#endif
/* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
if (!g_pfnrtNtKeFlushQueuedDpcs)
return VERR_NOT_SUPPORTED;
pArgs = (PRTMPARGS)ExAllocatePoolWithTag(NonPagedPool, MAXIMUM_PROCESSORS*sizeof(KDPC) + sizeof(RTMPARGS), (ULONG)'RTMp');
if (!pArgs)
return VERR_NO_MEMORY;
pArgs->pfnWorker = pfnWorker;
pArgs->pvUser1 = pvUser1;
pArgs->pvUser2 = pvUser2;
pArgs->idCpu = NIL_RTCPUID;
pArgs->cHits = 0;
paExecCpuDpcs = (KDPC *)(pArgs + 1);
if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
KeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
KeSetTargetProcessorDpc(&paExecCpuDpcs[0], (int)idCpu);
}
else
{
for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
{
KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
KeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
KeSetTargetProcessorDpc(&paExecCpuDpcs[i], i);
}
}
/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
*/
KIRQL oldIrql;
KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
/*
* We cannot do other than assume a 1:1 relationship between the
* affinity mask and the process despite the warnings in the docs.
* If someone knows a better way to get this done, please let bird know.
*/
if (enmCpuid == RT_NT_CPUID_SPECIFIC)
{
BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
Assert(ret);
}
else
{
unsigned iSelf = KeGetCurrentProcessorNumber();
for (unsigned i = 0; i < MAXIMUM_PROCESSORS; i++)
{
if ( (i != iSelf)
&& (Mask & RT_BIT_64(i)))
{
BOOLEAN ret = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
Assert(ret);
}
}
if (enmCpuid != RT_NT_CPUID_OTHERS)
pfnWorker(iSelf, pvUser1, pvUser2);
}
KeLowerIrql(oldIrql);
/* Flush all DPCs and wait for completion. (can take long!) */
/** @todo Consider changing this to an active wait using some atomic inc/dec
* stuff (and check for the current cpu above in the specific case). */
g_pfnrtNtKeFlushQueuedDpcs();
ExFreePool(pArgs);
return VINF_SUCCESS;
}
RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
return rtMpCall(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, 0);
}
RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
return rtMpCall(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, 0);
}
RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
if (!RTMpIsCpuOnline(idCpu))
return !RTMpIsCpuPossible(idCpu)
? VERR_CPU_NOT_FOUND
: VERR_CPU_OFFLINE;
return rtMpCall(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu);
}
static KDPC aPokeDpcs[MAXIMUM_PROCESSORS] = {0};
static bool fPokeDPCsInitialized = false;
static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
{
NOREF(Dpc);
NOREF(DeferredContext);
NOREF(SystemArgument1);
NOREF(SystemArgument2);
}
#ifndef IPRT_TARGET_NT4
ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
{
return 0;
}
int rtMpSendIpiVista(RTCPUID idCpu)
{
g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
//// g_pfnrtNtHalRequestIpi(1 << idCpu);
return VINF_SUCCESS;
}
int rtMpSendIpiWin7(RTCPUID idCpu)
{
g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
//// g_pfnrtNtHalSendSoftwareInterrupt(idCpu, DISPATCH_LEVEL);
return VINF_SUCCESS;
}
#endif /* IPRT_TARGET_NT4 */
int rtMpSendIpiDummy(RTCPUID idCpu)
{
return VERR_NOT_IMPLEMENTED;
}
RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
{
if (!RTMpIsCpuOnline(idCpu))
return !RTMpIsCpuPossible(idCpu)
? VERR_CPU_NOT_FOUND
: VERR_CPU_OFFLINE;
int rc = g_pfnrtSendIpi(idCpu);
if (rc == VINF_SUCCESS)
return rc;
/* Fallback. */
if (!fPokeDPCsInitialized)
{
for (unsigned i = 0; i < RT_ELEMENTS(aPokeDpcs); i++)
{
KeInitializeDpc(&aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
KeSetImportanceDpc(&aPokeDpcs[i], HighImportance);
KeSetTargetProcessorDpc(&aPokeDpcs[i], (int)i);
}
fPokeDPCsInitialized = true;
}
/* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
* KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
*/
KIRQL oldIrql;
KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
KeSetImportanceDpc(&aPokeDpcs[idCpu], HighImportance);
KeSetTargetProcessorDpc(&aPokeDpcs[idCpu], (int)idCpu);
/* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
* @note: not true on at least Vista & Windows 7
*/
BOOLEAN bRet = KeInsertQueueDpc(&aPokeDpcs[idCpu], 0, 0);
KeLowerIrql(oldIrql);
return (bRet == TRUE) ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
}