r0drv/linux/mp-r0drv-linux.c

	mp-r0drv-linux.c revision 2c8aed3abc13f4e22f66eda0e135c6197eea8003
236b2935f217749893b7034e59da3e3568928acevboxsync/* $Id$ */
236b2935f217749893b7034e59da3e3568928acevboxsync/** @file
236b2935f217749893b7034e59da3e3568928acevboxsync * IPRT - Multiprocessor, Ring-0 Driver, Linux.
236b2935f217749893b7034e59da3e3568928acevboxsync */
236b2935f217749893b7034e59da3e3568928acevboxsync
236b2935f217749893b7034e59da3e3568928acevboxsync/*
236b2935f217749893b7034e59da3e3568928acevboxsync * Copyright (C) 2008 Oracle Corporation
236b2935f217749893b7034e59da3e3568928acevboxsync *
236b2935f217749893b7034e59da3e3568928acevboxsync * This file is part of VirtualBox Open Source Edition (OSE), as
236b2935f217749893b7034e59da3e3568928acevboxsync * available from http://www.virtualbox.org. This file is free software;
236b2935f217749893b7034e59da3e3568928acevboxsync * you can redistribute it and/or modify it under the terms of the GNU
236b2935f217749893b7034e59da3e3568928acevboxsync * General Public License (GPL) as published by the Free Software
236b2935f217749893b7034e59da3e3568928acevboxsync * Foundation, in version 2 as it comes in the "COPYING" file of the
236b2935f217749893b7034e59da3e3568928acevboxsync * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
236b2935f217749893b7034e59da3e3568928acevboxsync * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
236b2935f217749893b7034e59da3e3568928acevboxsync *
236b2935f217749893b7034e59da3e3568928acevboxsync * The contents of this file may alternatively be used under the terms
236b2935f217749893b7034e59da3e3568928acevboxsync * of the Common Development and Distribution License Version 1.0
236b2935f217749893b7034e59da3e3568928acevboxsync * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
236b2935f217749893b7034e59da3e3568928acevboxsync * VirtualBox OSE distribution, in which case the provisions of the
236b2935f217749893b7034e59da3e3568928acevboxsync * CDDL are applicable instead of those of the GPL.
236b2935f217749893b7034e59da3e3568928acevboxsync *
236b2935f217749893b7034e59da3e3568928acevboxsync * You may elect to license modified versions of this file under the
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync * terms and conditions of either the GPL or the CDDL or both.
236b2935f217749893b7034e59da3e3568928acevboxsync */
236b2935f217749893b7034e59da3e3568928acevboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync/*******************************************************************************
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync*   Header Files                                                               *
236b2935f217749893b7034e59da3e3568928acevboxsync*******************************************************************************/
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include "the-linux-kernel.h"
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include "internal/iprt.h"
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include <iprt/mp.h>
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include <iprt/cpuset.h>
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include <iprt/err.h>
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include <iprt/asm.h>
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include <iprt/thread.h>
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#include "r0drv/mp-r0drv.h"
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
236b2935f217749893b7034e59da3e3568928acevboxsync
236b2935f217749893b7034e59da3e3568928acevboxsyncRTDECL(RTCPUID) RTMpCpuId(void)
236b2935f217749893b7034e59da3e3568928acevboxsync{
236b2935f217749893b7034e59da3e3568928acevboxsync    return smp_processor_id();
236b2935f217749893b7034e59da3e3568928acevboxsync}
236b2935f217749893b7034e59da3e3568928acevboxsyncRT_EXPORT_SYMBOL(RTMpCpuId);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync{
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return idCpu < RTCPUSET_MAX_CPUS && idCpu < NR_CPUS ? (int)idCpu : -1;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync}
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRT_EXPORT_SYMBOL(RTMpCpuIdToSetIndex);
236b2935f217749893b7034e59da3e3568928acevboxsync
236b2935f217749893b7034e59da3e3568928acevboxsync
236b2935f217749893b7034e59da3e3568928acevboxsyncRTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
236b2935f217749893b7034e59da3e3568928acevboxsync{
236b2935f217749893b7034e59da3e3568928acevboxsync    return iCpu < NR_CPUS ? (RTCPUID)iCpu : NIL_RTCPUID;
236b2935f217749893b7034e59da3e3568928acevboxsync}
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRT_EXPORT_SYMBOL(RTMpCpuIdFromSetIndex);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRTDECL(RTCPUID) RTMpGetMaxCpuId(void)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync{
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return NR_CPUS - 1; //???
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync}
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRT_EXPORT_SYMBOL(RTMpGetMaxCpuId);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync{
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#if defined(CONFIG_SMP)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    if (RT_UNLIKELY(idCpu >= NR_CPUS))
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync        return false;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# if defined(cpu_possible)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return cpu_possible(idCpu);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# else /* < 2.5.29 */
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return idCpu < (RTCPUID)smp_num_cpus;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# endif
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#else
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return idCpu == RTMpCpuId();
bc5cd42756b3f98351040bbfccc08dd9bacd103avboxsync#endif
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync}
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRT_EXPORT_SYMBOL(RTMpIsCpuPossible);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync{
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    RTCPUID idCpu;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    RTCpuSetEmpty(pSet);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    idCpu = RTMpGetMaxCpuId();
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    do
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    {
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync        if (RTMpIsCpuPossible(idCpu))
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync            RTCpuSetAdd(pSet, idCpu);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    } while (idCpu-- > 0);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return pSet;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync}
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRT_EXPORT_SYMBOL(RTMpGetSet);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRTDECL(RTCPUID) RTMpGetCount(void)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync{
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#ifdef CONFIG_SMP
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# if defined(CONFIG_HOTPLUG_CPU) /* introduced & uses cpu_present */
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return num_present_cpus();
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# elif defined(num_possible_cpus)
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return num_possible_cpus();
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 0)
bc5cd42756b3f98351040bbfccc08dd9bacd103avboxsync    return smp_num_cpus;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# else
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    RTCPUSET Set;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    RTMpGetSet(&Set);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return RTCpuSetCount(&Set);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync# endif
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#else
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync    return 1;
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync#endif
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync}
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRT_EXPORT_SYMBOL(RTMpGetCount);
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsync
5067a9619d7131c54d4ebb371d9dac91abdd34f6vboxsyncRTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
236b2935f217749893b7034e59da3e3568928acevboxsync{
236b2935f217749893b7034e59da3e3568928acevboxsync#ifdef CONFIG_SMP
    if (RT_UNLIKELY(idCpu >= NR_CPUS))
        return false;
# ifdef cpu_online
    return cpu_online(idCpu);
# else /* 2.4: */
    return cpu_online_map & RT_BIT_64(idCpu);
# endif
#else
    return idCpu == RTMpCpuId();
#endif
}
RT_EXPORT_SYMBOL(RTMpIsCpuOnline);


RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
{
#ifdef CONFIG_SMP
    RTCPUID idCpu;

    RTCpuSetEmpty(pSet);
    idCpu = RTMpGetMaxCpuId();
    do
    {
        if (RTMpIsCpuOnline(idCpu))
            RTCpuSetAdd(pSet, idCpu);
    } while (idCpu-- > 0);
#else
    RTCpuSetEmpty(pSet);
    RTCpuSetAdd(pSet, RTMpCpuId());
#endif
    return pSet;
}
RT_EXPORT_SYMBOL(RTMpGetOnlineSet);


RTDECL(RTCPUID) RTMpGetOnlineCount(void)
{
#ifdef CONFIG_SMP
# if defined(num_online_cpus)
    return num_online_cpus();
# else
    RTCPUSET Set;
    RTMpGetOnlineSet(&Set);
    return RTCpuSetCount(&Set);
# endif
#else
    return 1;
#endif
}
RT_EXPORT_SYMBOL(RTMpGetOnlineCount);


RTDECL(bool) RTMpIsCpuWorkPending(void)
{
    /** @todo (not used on non-Windows platforms yet). */
    return false;
}
RT_EXPORT_SYMBOL(RTMpIsCpuWorkPending);


/**
 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
 *
 * @param   pvInfo      Pointer to the RTMPARGS package.
 */
static void rtmpLinuxWrapper(void *pvInfo)
{
    PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
    ASMAtomicIncU32(&pArgs->cHits);
    pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
}


RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
    int rc;
    RTMPARGS Args;

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0)
    RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
#endif
    Args.pfnWorker = pfnWorker;
    Args.pvUser1 = pvUser1;
    Args.pvUser2 = pvUser2;
    Args.idCpu = NIL_RTCPUID;
    Args.cHits = 0;

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
    rc = on_each_cpu(rtmpLinuxWrapper, &Args, 1 /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0)
    rc = on_each_cpu(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#else /* older kernels */
    RTThreadPreemptDisable(&PreemptState);
    rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
    local_irq_disable();
    rtmpLinuxWrapper(&Args);
    local_irq_enable();
    RTThreadPreemptRestore(&PreemptState);
#endif /* older kernels */
    Assert(rc == 0); NOREF(rc);
    return VINF_SUCCESS;
}
RT_EXPORT_SYMBOL(RTMpOnAll);


RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
    int rc;
    RTMPARGS Args;

    RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
    Args.pfnWorker = pfnWorker;
    Args.pvUser1 = pvUser1;
    Args.pvUser2 = pvUser2;
    Args.idCpu = NIL_RTCPUID;
    Args.cHits = 0;

    RTThreadPreemptDisable(&PreemptState);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
    rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
#else /* older kernels */
    rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#endif /* older kernels */
    RTThreadPreemptRestore(&PreemptState);

    Assert(rc == 0); NOREF(rc);
    return VINF_SUCCESS;
}
RT_EXPORT_SYMBOL(RTMpOnOthers);


#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
/**
 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
 * employed by RTMpOnSpecific on older kernels that lacks smp_call_function_single.
 *
 * @param   pvInfo      Pointer to the RTMPARGS package.
 */
static void rtmpOnSpecificLinuxWrapper(void *pvInfo)
{
    PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
    RTCPUID idCpu = RTMpCpuId();

    if (idCpu == pArgs->idCpu)
    {
        pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
        ASMAtomicIncU32(&pArgs->cHits);
    }
}
#endif


RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
    int rc;
    RTMPARGS Args;

    RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
    Args.pfnWorker = pfnWorker;
    Args.pvUser1 = pvUser1;
    Args.pvUser2 = pvUser2;
    Args.idCpu = idCpu;
    Args.cHits = 0;

    if (!RTMpIsCpuPossible(idCpu))
        return VERR_CPU_NOT_FOUND;

    RTThreadPreemptDisable(&PreemptState);
    if (idCpu != RTMpCpuId())
    {
        if (RTMpIsCpuOnline(idCpu))
        {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
            rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 1 /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
            rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#else /* older kernels */
            rc = smp_call_function(rtmpOnSpecificLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
#endif /* older kernels */
            Assert(rc == 0);
            rc = Args.cHits ? VINF_SUCCESS : VERR_CPU_OFFLINE;
        }
        else
            rc = VERR_CPU_OFFLINE;
    }
    else
    {
        rtmpLinuxWrapper(&Args);
        rc = VINF_SUCCESS;
    }
    RTThreadPreemptRestore(&PreemptState);;

    NOREF(rc);
    return rc;
}
RT_EXPORT_SYMBOL(RTMpOnSpecific);


#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
/**
 * Dummy callback used by RTMpPokeCpu.
 *
 * @param   pvInfo      Ignored.
 */
static void rtmpLinuxPokeCpuCallback(void *pvInfo)
{
    NOREF(pvInfo);
}
#endif


RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
    int rc;

    if (!RTMpIsCpuPossible(idCpu))
        return VERR_CPU_NOT_FOUND;
    if (!RTMpIsCpuOnline(idCpu))
        return VERR_CPU_OFFLINE;

# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
    rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* wait */);
# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
    rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* retry */, 0 /* wait */);
# else  /* older kernels */
#  error oops
# endif /* older kernels */
    NOREF(rc);
    Assert(rc == 0);
    return VINF_SUCCESS;

#else  /* older kernels */
    /* no unicast here? */
    return VERR_NOT_SUPPORTED;
#endif /* older kernels */
}
RT_EXPORT_SYMBOL(RTMpPokeCpu);