diff options
Diffstat (limited to 'src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp')
| -rw-r--r-- | src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp | 1996 |
1 files changed, 1996 insertions, 0 deletions
diff --git a/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp b/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp new file mode 100644 index 00000000..a2b27be2 --- /dev/null +++ b/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp @@ -0,0 +1,1996 @@ +/* $Id: mp-r0drv-nt.cpp $ */ +/** @file + * IPRT - Multiprocessor, Ring-0 Driver, NT. + */ + +/* + * Copyright (C) 2008-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <https://www.gnu.org/licenses>. + * + * The contents of this file may alternatively be used under the terms + * of the Common Development and Distribution License Version 1.0 + * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included + * in the VirtualBox distribution, in which case the provisions of the + * CDDL are applicable instead of those of the GPL. + * + * You may elect to license modified versions of this file under the + * terms and conditions of either the GPL or the CDDL or both. + * + * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#include "the-nt-kernel.h" + +#include <iprt/mp.h> +#include <iprt/cpuset.h> +#include <iprt/err.h> +#include <iprt/asm.h> +#include <iprt/log.h> +#include <iprt/mem.h> +#include <iprt/time.h> +#include "r0drv/mp-r0drv.h" +#include "symdb.h" +#include "internal-r0drv-nt.h" +#include "internal/mp.h" + + +/********************************************************************************************************************************* +* Structures and Typedefs * +*********************************************************************************************************************************/ +typedef enum +{ + RT_NT_CPUID_SPECIFIC, + RT_NT_CPUID_PAIR, + RT_NT_CPUID_OTHERS, + RT_NT_CPUID_ALL +} RT_NT_CPUID; + + +/** + * Used by the RTMpOnSpecific. + */ +typedef struct RTMPNTONSPECIFICARGS +{ + /** Set if we're executing. */ + bool volatile fExecuting; + /** Set when done executing. */ + bool volatile fDone; + /** Number of references to this heap block. */ + uint32_t volatile cRefs; + /** Event that the calling thread is waiting on. */ + KEVENT DoneEvt; + /** The deferred procedure call object. */ + KDPC Dpc; + /** The callback argument package. */ + RTMPARGS CallbackArgs; +} RTMPNTONSPECIFICARGS; +/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */ +typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS; + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */ +#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31) + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** Maximum number of processor groups. */ +uint32_t g_cRtMpNtMaxGroups; +/** Maximum number of processors. */ +uint32_t g_cRtMpNtMaxCpus; +/** Number of active processors. */ +uint32_t volatile g_cRtMpNtActiveCpus; +/** The NT CPU set. + * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll + * have to cache it. Fortunately, NT doesn't really support taking CPUs offline, + * and taking them online was introduced with W2K8 where it is intended for virtual + * machines and not real HW. We update this, g_cRtMpNtActiveCpus and + * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback. + */ +RTCPUSET g_rtMpNtCpuSet; + +/** Static per group info. + * @remarks With 256 groups this takes up 33KB. */ +static struct +{ + /** The max CPUs in the group. */ + uint16_t cMaxCpus; + /** The number of active CPUs at the time of initialization. */ + uint16_t cActiveCpus; + /** CPU set indexes for each CPU in the group. */ + int16_t aidxCpuSetMembers[64]; +} g_aRtMpNtCpuGroups[256]; +/** Maps CPU set indexes to RTCPUID. + * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them + * and shuffle duplicates during CPU hotplugging. We assign temporary IDs to + * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active + * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */ +RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS]; +/** The handle of the rtR0NtMpProcessorChangeCallback registration. */ +static PVOID g_pvMpCpuChangeCallback = NULL; +/** Size of the KAFFINITY_EX structure. + * This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release + * (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */ +static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap) + + RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256; +/** The size value of the KAFFINITY_EX structure. */ +static uint16_t g_cRtMpNtKaffinityExEntries = 256; + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx, + PNTSTATUS prcOperationStatus); +static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo); + + + +/** + * Initalizes multiprocessor globals (called by rtR0InitNative). + * + * @returns IPRT status code. + * @param pOsVerInfo Version information. + */ +DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo) +{ +#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \ + AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 ) +#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \ + AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet) +#define MY_CHECK(a_Check, a_DbgPrintArgs) \ + AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 ) + + /* + * API combination checks. + */ + MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex, + ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"), + VERR_SYMBOL_NOT_FOUND); + + /* + * Get max number of processor groups. + * + * We may need to upadjust this number below, because windows likes to keep + * all options open when it comes to hotplugged CPU group assignments. A + * server advertising up to 64 CPUs in the ACPI table will get a result of + * 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows + * server 2012 does a two processor group setup for it, the sum of the + * GroupInfo[*].MaximumProcessorCount members below is 128. This is probably + * because windows doesn't want to make decisions grouping of hotpluggable CPUs. + * So, we need to bump the maximum count to 128 below do deal with this as we + * want to have valid CPU set indexes for all potential CPUs - how could we + * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet()) + * should equal RTMpGetCount(). + */ + if (g_pfnrtKeQueryMaximumGroupCount) + { + g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount(); + MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0, + ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS), + VERR_MP_TOO_MANY_CPUS); + } + else + g_cRtMpNtMaxGroups = 1; + + /* + * Get max number CPUs. + * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET. + */ + if (g_pfnrtKeQueryMaximumProcessorCountEx) + { + g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS); + MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0, + ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n", + g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS), + VERR_MP_TOO_MANY_CPUS); + } + else if (g_pfnrtKeQueryMaximumProcessorCount) + { + g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount(); + MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0, + ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n", + g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS), + VERR_MP_TOO_MANY_CPUS); + } + else if (g_pfnrtKeQueryActiveProcessors) + { + KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors(); + MY_CHECK_RETURN(fActiveProcessors != 0, + ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"), + VERR_INTERNAL_ERROR_2); + g_cRtMpNtMaxCpus = 0; + do + { + g_cRtMpNtMaxCpus++; + fActiveProcessors >>= 1; + } while (fActiveProcessors); + } + else + g_cRtMpNtMaxCpus = KeNumberProcessors; + + /* + * Just because we're a bit paranoid about getting something wrong wrt to the + * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx + * and KeQueryLogicalProcessorRelationship information to match up. + */ + for (unsigned cTries = 0;; cTries++) + { + /* + * Get number of active CPUs. + */ + if (g_pfnrtKeQueryActiveProcessorCountEx) + { + g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS); + MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0, + ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n", + g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus), + VERR_MP_TOO_MANY_CPUS); + } + else if (g_pfnrtKeQueryActiveProcessorCount) + { + g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL); + MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0, + ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n", + g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus), + VERR_MP_TOO_MANY_CPUS); + } + else + g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus; + + /* + * Query the details for the groups to figure out which CPUs are online as + * well as the NT index limit. + */ + for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++) +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID; +#else + g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID; +#endif + for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++) + { + g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0; + g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0; + for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++) + g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1; + } + + if (g_pfnrtKeQueryLogicalProcessorRelationship) + { + MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber, + ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"), + VERR_SYMBOL_NOT_FOUND); + MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex, + ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"), + VERR_SYMBOL_NOT_FOUND); + MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx, + ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"), + VERR_SYMBOL_NOT_FOUND); + + SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL; + int rc = rtR0NtInitQueryGroupRelations(&pInfo); + if (RT_FAILURE(rc)) + return rc; + + MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups, + ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n", + pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups)); + MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups, + ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n", + pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups)); + + /* + * First we need to recalc g_cRtMpNtMaxCpus (see above). + */ + uint32_t cMaxCpus = 0; + uint32_t idxGroup; + for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++) + { + const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup]; + MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP, + ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount)); + MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount, + ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n", + pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount)); + cMaxCpus += pGrpInfo->MaximumProcessorCount; + } + if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc)) + { + DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus); +#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx)); + while (i-- > g_cRtMpNtMaxCpus) + g_aidRtMpNtByCpuSetIdx[i] = i; +#endif + g_cRtMpNtMaxCpus = cMaxCpus; + if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS) + { + MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0, + ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS)); + rc = VERR_MP_TOO_MANY_CPUS; + } + } + + /* + * Calc online mask, partition IDs and such. + * + * Also check ASSUMPTIONS: + * + * 1. Processor indexes going from 0 and up to + * KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1. + * + * 2. Currently valid processor indexes, i.e. accepted by + * KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes + * from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1. + * + * 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of + * relevant bits in the ActiveProcessorMask (from LSB). + * + * 4. Active processor count found in KeQueryLogicalProcessorRelationship + * output matches what KeQueryActiveProcessorCountEx(ALL) returns. + * + * 5. Active + inactive processor counts in same does not exceed + * KeQueryMaximumProcessorCountEx(ALL). + * + * Note! Processor indexes are assigned as CPUs come online and are not + * preallocated according to group maximums. Since CPUS are only taken + * online and never offlined, this means that internal CPU bitmaps are + * never sparse and no time is wasted scanning unused bits. + * + * Unfortunately, it means that ring-3 cannot easily guess the index + * assignments when hotswapping is used, and must use GIP when available. + */ + RTCpuSetEmpty(&g_rtMpNtCpuSet); + uint32_t cInactive = 0; + uint32_t cActive = 0; + uint32_t idxCpuMax = 0; + uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1; + for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++) + { + const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup]; + MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP, + ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount)); + MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount, + ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n", + pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount)); + + g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount; + g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount; + + for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++) + { + PROCESSOR_NUMBER ProcNum; + ProcNum.Group = (USHORT)idxGroup; + ProcNum.Number = (UCHAR)idxMember; + ProcNum.Reserved = 0; + ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum); + if (idxCpu != INVALID_PROCESSOR_INDEX) + { + MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */ + ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n", + idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS)); + if (idxCpu > idxCpuMax) + idxCpuMax = idxCpu; + g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu; +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember); +#endif + + ProcNum.Group = UINT16_MAX; + ProcNum.Number = UINT8_MAX; + ProcNum.Reserved = UINT8_MAX; + NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum); + MY_CHECK_BREAK(NT_SUCCESS(rcNt), + ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt)); + MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember, + ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n", + idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember)); + + if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)) + { + RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu); + cActive++; + } + else + cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */ + } + else + { + /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */ + MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)), + ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n", + idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount, + pGrpInfo->ActiveProcessorMask)); + cInactive++; + if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus) + { + g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive; +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember) + | RTMPNT_ID_F_INACTIVE; +#endif + idxCpuSetNextInactive--; + } + } + } + } + + MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */ + ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus)); + + /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as + best as we can by as best we can by stipulating maximum member counts + from the previous group. */ + if ( RT_SUCCESS(rc) + && idxGroup < pInfo->Group.MaximumGroupCount) + { + uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive); + while (idxGroup < pInfo->Group.MaximumGroupCount) + { + uint32_t cMaxMembers = 0; + if (g_pfnrtKeQueryMaximumProcessorCountEx) + cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup); + if (cMaxMembers != 0 || cInactiveLeft == 0) + AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft); + else + { + uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup; + cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount; + while (cMaxMembers * cGroupsLeft < cInactiveLeft) + cMaxMembers++; + if (cMaxMembers > cInactiveLeft) + cMaxMembers = cInactiveLeft; + } + + g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers; + g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0; + for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++) + if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus) + { + g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive; +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember) + | RTMPNT_ID_F_INACTIVE; +#endif + idxCpuSetNextInactive--; + } + cInactiveLeft -= cMaxMembers; + idxGroup++; + } + } + + /* We're done with pInfo now, free it so we can start returning when assertions fail. */ + RTMemFree(pInfo); + if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */ + return rc; + MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus, + ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus), + VERR_INTERNAL_ERROR_3); + MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */ + ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n", + idxCpuMax, cActive), + VERR_INTERNAL_ERROR_4); + + /* Retry if CPUs were added. */ + if ( cActive != g_cRtMpNtActiveCpus + && cTries < 16) + continue; + MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */ + ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus), + VERR_INTERNAL_ERROR_5); + } + else + { + /* Legacy: */ + MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"), + VERR_SYMBOL_NOT_FOUND); + + /** @todo Is it possible that the affinity mask returned by + * KeQueryActiveProcessors is sparse? */ + if (g_pfnrtKeQueryActiveProcessors) + RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors()); + else if (g_cRtMpNtMaxCpus < 64) + RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1); + else + { + MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus), + VERR_MP_TOO_MANY_CPUS); + RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX); + } + + g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus; + g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus; + for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++) + { + g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i; +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i); +#endif + } + } + + /* + * Register CPU hot plugging callback (it also counts active CPUs). + */ + Assert(g_pvMpCpuChangeCallback == NULL); + if (g_pfnrtKeRegisterProcessorChangeCallback) + { + MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback, + ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"), + VERR_SYMBOL_NOT_FOUND); + + RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet; + RTCpuSetEmpty(&g_rtMpNtCpuSet); + uint32_t const cActiveCpus = g_cRtMpNtActiveCpus; + g_cRtMpNtActiveCpus = 0; + + g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/, + KE_PROCESSOR_CHANGE_ADD_EXISTING); + if (g_pvMpCpuChangeCallback) + { + if (cActiveCpus == g_cRtMpNtActiveCpus) + { /* likely */ } + else + { + g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback); + if (cTries < 16) + { + /* Retry if CPUs were added. */ + MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus, + ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n", + g_cRtMpNtActiveCpus, cActiveCpus), + VERR_INTERNAL_ERROR_2); + MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus, + ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n", + g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus), + VERR_INTERNAL_ERROR_2); + continue; + } + MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus), + VERR_INTERNAL_ERROR_3); + } + } + else + { + AssertFailed(); + g_rtMpNtCpuSet = ActiveSetCopy; + g_cRtMpNtActiveCpus = cActiveCpus; + } + } + break; + } /* Retry loop for stable active CPU count. */ + +#undef MY_CHECK_RETURN + + /* + * Special IPI fun for RTMpPokeCpu. + * + * On Vista and later the DPC method doesn't seem to reliably send IPIs, + * so we have to use alternative methods. + * + * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on + * W10+), it looks faster and more convenient to use, however we're either + * using it wrong or it doesn't reliably do what we want (see @bugref{8343}). + * + * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc + * for doing targetted IPIs. Trouble with this API is that it changed + * fundamentally in Window 7 when they added support for lots of processors. + * + * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI + * API KeIpiGenericCall. + */ + if ( pOsVerInfo->uMajorVer > 6 + || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0)) + g_pfnrtHalRequestIpiPreW7 = NULL; + else + g_pfnrtHalRequestIpiW7Plus = NULL; + + if ( g_pfnrtHalRequestIpiW7Plus + && g_pfnrtKeInitializeAffinityEx + && g_pfnrtKeAddProcessorAffinityEx + && g_pfnrtKeGetProcessorIndexFromNumber) + { + /* Determine the real size of the KAFFINITY_EX structure. */ + size_t const cbAffinity = _8K; + PKAFFINITY_EX pAffinity = (PKAFFINITY_EX)RTMemAllocZ(cbAffinity); + AssertReturn(pAffinity, VERR_NO_MEMORY); + size_t const cMaxEntries = (cbAffinity - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(pAffinity->Bitmap[0]); + g_pfnrtKeInitializeAffinityEx(pAffinity); + if (pAffinity->Size > 1 && pAffinity->Size <= cMaxEntries) + { + g_cRtMpNtKaffinityExEntries = pAffinity->Size; + g_cbRtMpNtKaffinityEx = pAffinity->Size * sizeof(pAffinity->Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]); + g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus; + RTMemFree(pAffinity); + DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n"); + return VINF_SUCCESS; + } + DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! pAffinity->Size=%u\n", pAffinity->Size); + AssertReleaseMsg(pAffinity->Size <= cMaxEntries, ("%#x\n", pAffinity->Size)); /* stack is toast if larger (32768 CPUs). */ + RTMemFree(pAffinity); + } + + if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall) + { + DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n"); + g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi; + } + else if (g_pfnrtKeSetTargetProcessorDpc) + { + DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n"); + g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc; + /* Windows XP should send always send an IPI -> VERIFY */ + } + else + { + DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n"); + Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50); + g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported; + } + + return VINF_SUCCESS; +} + + +/** + * Called by rtR0TermNative. + */ +DECLHIDDEN(void) rtR0MpNtTerm(void) +{ + /* + * Deregister the processor change callback. + */ + PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback; + g_pvMpCpuChangeCallback = NULL; + if (pvMpCpuChangeCallback) + { + AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback); + g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback); + } +} + + +DECLHIDDEN(int) rtR0MpNotificationNativeInit(void) +{ + return VINF_SUCCESS; +} + + +DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void) +{ +} + + +/** + * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function. + * + * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When + * registered, it's called for each active CPU in the system, avoiding racing + * CPU hotplugging (as well as testing the callback). + * + * @param pvUser User context (not used). + * @param pChangeCtx Change context (in). + * @param prcOperationStatus Operation status (in/out). + * + * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify + * notification callbacks. At least during callback registration + * callout, we're owning KiDynamicProcessorLock. + * + * @remarks When registering the handler, we first get KeProcessorAddStartNotify + * callbacks for all active CPUs, and after they all succeed we get the + * KeProcessorAddCompleteNotify callbacks. + */ +static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx, + PNTSTATUS prcOperationStatus) +{ + RT_NOREF(pvUser, prcOperationStatus); + switch (pChangeCtx->State) + { + /* + * Check whether we can deal with the CPU, failing the start operation if we + * can't. The checks we are doing here are to avoid complicated/impossible + * cases in KeProcessorAddCompleteNotify. They are really just verify specs. + */ + case KeProcessorAddStartNotify: + { + NTSTATUS rcNt = STATUS_SUCCESS; + if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS) + { + if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus) + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n", + pChangeCtx->NtNumber, g_cRtMpNtMaxCpus); + rcNt = STATUS_INTERNAL_ERROR; + } + + /* The ProcessNumber field was introduced in Windows 7. */ + PROCESSOR_NUMBER ProcNum; + if (g_pfnrtKeGetProcessorIndexFromNumber) + { + ProcNum = pChangeCtx->ProcNumber; + KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum); + if (idxCpu != pChangeCtx->NtNumber) + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n", + ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber); + rcNt = STATUS_INTERNAL_ERROR; + } + } + else + { + ProcNum.Group = 0; + ProcNum.Number = pChangeCtx->NtNumber; + } + + if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups) + && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers)) + { + if (ProcNum.Group >= g_cRtMpNtMaxGroups) + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n", + ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups); + rcNt = STATUS_INTERNAL_ERROR; + } + + if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus) + { + Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1); + if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1) + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n", + ProcNum.Group, ProcNum.Number); + rcNt = STATUS_INTERNAL_ERROR; + } + + Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID); + if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID) + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n", + pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number); + rcNt = STATUS_INTERNAL_ERROR; + } + } + else + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n", + ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number); + rcNt = STATUS_INTERNAL_ERROR; + } + } + else + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n", + ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers)); + rcNt = STATUS_INTERNAL_ERROR; + } + } + else + { + DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n", + pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS); + rcNt = STATUS_INTERNAL_ERROR; + } + if (!NT_SUCCESS(rcNt)) + *prcOperationStatus = rcNt; + break; + } + + /* + * Update the globals. Since we've checked out range limits and other + * limitations already we just AssertBreak here. + */ + case KeProcessorAddCompleteNotify: + { + /* + * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify. + */ + AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS); + AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus); + Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */ + PROCESSOR_NUMBER ProcNum; + if (g_pfnrtKeGetProcessorIndexFromNumber) + { + ProcNum = pChangeCtx->ProcNumber; + AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber); + AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)); + AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups); + } + else + { + ProcNum.Group = 0; + ProcNum.Number = pChangeCtx->NtNumber; + } + AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers)); + AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus); + AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1); + AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID); + + /* + * Add ourselves to the online CPU set and update the active CPU count. + */ + RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber); + ASMAtomicIncU32(&g_cRtMpNtActiveCpus); + + /* + * Update the group info. + * + * If the index prediction failed (real hotplugging callbacks only) we + * have to switch it around. This is particularly annoying when we + * use the index as the ID. + */ +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); + RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber]; + if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu) + { + Assert(idOld & RTMPNT_ID_F_INACTIVE); + int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number]; + g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest; + g_aidRtMpNtByCpuSetIdx[idxDest] = idOld; + } + g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu; +#else + Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber); + int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number]; + if ((ULONG)idxDest != pChangeCtx->NtNumber) + { + bool fFound = false; + uint32_t idxOldGroup = g_cRtMpNtMaxGroups; + while (idxOldGroup-- > 0 && !fFound) + { + uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus; + while (idxMember-- > 0) + if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber) + { + g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest; + fFound = true; + break; + } + } + Assert(fFound); + } +#endif + g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber; + + /* + * Do MP notification callbacks. + */ + rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber); + break; + } + + case KeProcessorAddFailureNotify: + /* ignore */ + break; + + default: + AssertMsgFailed(("State=%u\n", pChangeCtx->State)); + } +} + + +/** + * Wrapper around KeQueryLogicalProcessorRelationship. + * + * @returns IPRT status code. + * @param ppInfo Where to return the info. Pass to RTMemFree when done. + */ +static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo) +{ + ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) + + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP); + NTSTATUS rcNt; + do + { + SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo); + if (pInfo) + { + rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo); + if (NT_SUCCESS(rcNt)) + { + *ppInfo = pInfo; + return VINF_SUCCESS; + } + + RTMemFree(pInfo); + pInfo = NULL; + } + else + rcNt = STATUS_NO_MEMORY; + } while (rcNt == STATUS_INFO_LENGTH_MISMATCH); + DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt); + AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt)); + return RTErrConvertFromNtStatus(rcNt); +} + + + + + +RTDECL(RTCPUID) RTMpCpuId(void) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + PROCESSOR_NUMBER ProcNum; + ProcNum.Group = 0; + if (g_pfnrtKeGetCurrentProcessorNumberEx) + { + ProcNum.Number = 0; + g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum); + } + else + ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */ + return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); + +#else + + if (g_pfnrtKeGetCurrentProcessorNumberEx) + { + KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL); + Assert(idxCpu < RTCPUSET_MAX_CPUS); + return idxCpu; + } + + return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */ +#endif +} + + +RTDECL(int) RTMpCurSetIndex(void) +{ +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + if (g_pfnrtKeGetCurrentProcessorNumberEx) + { + KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL); + Assert(idxCpu < RTCPUSET_MAX_CPUS); + return idxCpu; + } + return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */ +#else + return (int)RTMpCpuId(); +#endif +} + + +RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu) +{ +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 }; + KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum); + Assert(idxCpu < RTCPUSET_MAX_CPUS); + *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); + return idxCpu; +#else + return *pidCpu = RTMpCpuId(); +#endif +} + + +RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu) +{ +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + if (idCpu != NIL_RTCPUID) + { + if (g_pfnrtKeGetProcessorIndexFromNumber) + { + PROCESSOR_NUMBER ProcNum; + ProcNum.Group = rtMpCpuIdGetGroup(idCpu); + ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu); + ProcNum.Reserved = 0; + KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum); + if (idxCpu != INVALID_PROCESSOR_INDEX) + { + Assert(idxCpu < g_cRtMpNtMaxCpus); + Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu); + return idxCpu; + } + + /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index + mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */ + if ( ProcNum.Group < g_cRtMpNtMaxGroups + && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus) + return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number]; + } + else if (rtMpCpuIdGetGroup(idCpu) == 0) + return rtMpCpuIdGetGroupMember(idCpu); + } + return -1; +#else + /* 1:1 mapping, just do range checks. */ + return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1; +#endif +} + + +RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu) +{ +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + if ((unsigned)iCpu < g_cRtMpNtMaxCpus) + { + if (g_pfnrtKeGetProcessorIndexFromNumber) + { + PROCESSOR_NUMBER ProcNum = { 0, 0, 0 }; + NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum); + if (NT_SUCCESS(rcNt)) + { + Assert(ProcNum.Group <= g_cRtMpNtMaxGroups); + Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE) + == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number)); + return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number); + } + } + return g_aidRtMpNtByCpuSetIdx[iCpu]; + } + return NIL_RTCPUID; +#else + /* 1:1 mapping, just do range checks. */ + return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID; +#endif +} + + +RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + if (idxGroup < g_cRtMpNtMaxGroups) + if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus) + return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember]; + return -1; +} + + +RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive) +{ + if (idxGroup < g_cRtMpNtMaxGroups) + { + if (pcActive) + *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus; + return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus; + } + if (pcActive) + *pcActive = 0; + return 0; +} + + +RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void) +{ + return g_cRtMpNtMaxGroups; +} + + +RTDECL(RTCPUID) RTMpGetMaxCpuId(void) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1); +#else + /* According to MSDN the processor indexes goes from 0 to the maximum + number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */ + return g_cRtMpNtMaxCpus - 1; +#endif +} + + +RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu); +} + + +RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + +#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER + if (idCpu != NIL_RTCPUID) + { + unsigned idxGroup = rtMpCpuIdGetGroup(idCpu); + if (idxGroup < g_cRtMpNtMaxGroups) + return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus; + } + return false; + +#else + /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see + comment in RTMpGetMaxCpuId). */ + return idCpu < g_cRtMpNtMaxCpus; +#endif +} + + + +RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + /* The set of possible CPU IDs(/indexes) are from 0 up to + g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */ + RTCpuSetEmpty(pSet); + int idxCpu = g_cRtMpNtMaxCpus; + while (idxCpu-- > 0) + RTCpuSetAddByIndex(pSet, idxCpu); + return pSet; +} + + +RTDECL(RTCPUID) RTMpGetCount(void) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + return g_cRtMpNtMaxCpus; +} + + +RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + *pSet = g_rtMpNtCpuSet; + return pSet; +} + + +RTDECL(RTCPUID) RTMpGetOnlineCount(void) +{ + RTCPUSET Set; + RTMpGetOnlineSet(&Set); + return RTCpuSetCount(&Set); +} + + +RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void) +{ + /** @todo fix me */ + return RTMpGetOnlineCount(); +} + + + +#if 0 +/* Experiment with checking the undocumented KPRCB structure + * 'dt nt!_kprcb 0xaddress' shows the layout + */ +typedef struct +{ + LIST_ENTRY DpcListHead; + ULONG_PTR DpcLock; + volatile ULONG DpcQueueDepth; + ULONG DpcQueueCount; +} KDPC_DATA, *PKDPC_DATA; + +RTDECL(bool) RTMpIsCpuWorkPending(void) +{ + uint8_t *pkprcb; + PKDPC_DATA pDpcData; + + _asm { + mov eax, fs:0x20 + mov pkprcb, eax + } + pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0); + if (pDpcData->DpcQueueDepth) + return true; + + pDpcData++; + if (pDpcData->DpcQueueDepth) + return true; + return false; +} +#else +RTDECL(bool) RTMpIsCpuWorkPending(void) +{ + /** @todo not implemented */ + return false; +} +#endif + + +/** + * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for + * the RTMpOnAll case. + * + * @param uUserCtx The user context argument (PRTMPARGS). + */ +static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx) +{ + PRTMPARGS pArgs = (PRTMPARGS)uUserCtx; + /*ASMAtomicIncU32(&pArgs->cHits); - not needed */ + pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2); + return 0; +} + + +/** + * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for + * the RTMpOnOthers case. + * + * @param uUserCtx The user context argument (PRTMPARGS). + */ +static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx) +{ + PRTMPARGS pArgs = (PRTMPARGS)uUserCtx; + RTCPUID idCpu = RTMpCpuId(); + if (pArgs->idCpu != idCpu) + { + /*ASMAtomicIncU32(&pArgs->cHits); - not needed */ + pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2); + } + return 0; +} + + +/** + * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for + * the RTMpOnPair case. + * + * @param uUserCtx The user context argument (PRTMPARGS). + */ +static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx) +{ + PRTMPARGS pArgs = (PRTMPARGS)uUserCtx; + RTCPUID idCpu = RTMpCpuId(); + if ( pArgs->idCpu == idCpu + || pArgs->idCpu2 == idCpu) + { + ASMAtomicIncU32(&pArgs->cHits); + pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2); + } + return 0; +} + + +/** + * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for + * the RTMpOnSpecific case. + * + * @param uUserCtx The user context argument (PRTMPARGS). + */ +static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx) +{ + PRTMPARGS pArgs = (PRTMPARGS)uUserCtx; + RTCPUID idCpu = RTMpCpuId(); + if (pArgs->idCpu == idCpu) + { + ASMAtomicIncU32(&pArgs->cHits); + pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2); + } + return 0; +} + + +/** + * Internal worker for the RTMpOn* APIs using KeIpiGenericCall. + * + * @returns VINF_SUCCESS. + * @param pfnWorker The callback. + * @param pvUser1 User argument 1. + * @param pvUser2 User argument 2. + * @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks. + * @param idCpu First CPU to match, ultimately specific to the + * pfnNativeWrapper used. + * @param idCpu2 Second CPU to match, ultimately specific to the + * pfnNativeWrapper used. + * @param pcHits Where to return the number of this. Optional. + */ +static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2, + PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2, + uint32_t *pcHits) +{ + RTMPARGS Args; + Args.pfnWorker = pfnWorker; + Args.pvUser1 = pvUser1; + Args.pvUser2 = pvUser2; + Args.idCpu = idCpu; + Args.idCpu2 = idCpu2; + Args.cRefs = 0; + Args.cHits = 0; + + AssertPtr(g_pfnrtKeIpiGenericCall); + g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args); + if (pcHits) + *pcHits = Args.cHits; + return VINF_SUCCESS; +} + + +/** + * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER + * + * @param Dpc DPC object + * @param DeferredContext Context argument specified by KeInitializeDpc + * @param SystemArgument1 Argument specified by KeInsertQueueDpc + * @param SystemArgument2 Argument specified by KeInsertQueueDpc + */ +static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2) +{ + PRTMPARGS pArgs = (PRTMPARGS)DeferredContext; + RT_NOREF3(Dpc, SystemArgument1, SystemArgument2); + + ASMAtomicIncU32(&pArgs->cHits); + pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2); + + /* Dereference the argument structure. */ + int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs); + Assert(cRefs >= 0); + if (cRefs == 0) + RTMemFree(pArgs); +} + + +/** + * Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc. + * + * This is shared with the timer code. + * + * @returns IPRT status code (errors are asserted). + * @retval VERR_CPU_NOT_FOUND if impossible CPU. Not asserted. + * @param pDpc The DPC. + * @param idCpu The ID of the new target CPU. + * @note Callable at any IRQL. + */ +DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu) +{ + if (g_pfnrtKeSetTargetProcessorDpcEx) + { + /* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does + the reverse conversion internally). */ + PROCESSOR_NUMBER ProcNum; + NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum); + if (NT_SUCCESS(rcNt)) + { + rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum); + AssertLogRelMsgReturn(NT_SUCCESS(rcNt), + ("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt), + RTErrConvertFromNtStatus(rcNt)); + } + else if (rcNt == STATUS_INVALID_PARAMETER) + return VERR_CPU_NOT_FOUND; + else + AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt), + RTErrConvertFromNtStatus(rcNt)); + + } + else if (g_pfnrtKeSetTargetProcessorDpc) + g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu)); + else + return VERR_NOT_SUPPORTED; + return VINF_SUCCESS; +} + + +/** + * Internal worker for the RTMpOn* APIs. + * + * @returns IPRT status code. + * @param pfnWorker The callback. + * @param pvUser1 User argument 1. + * @param pvUser2 User argument 2. + * @param enmCpuid What to do / is idCpu valid. + * @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or + * RT_NT_CPUID_PAIR, otherwise ignored. + * @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored. + * @param pcHits Where to return the number of this. Optional. + */ +static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2, + RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits) +{ +#if 0 + /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the + * driver verifier doesn't complain... + */ + AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL)); +#endif + /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */ + if (!g_pfnrtNtKeFlushQueuedDpcs) + return VERR_NOT_SUPPORTED; + + /* + * Make a copy of the active CPU set and figure out how many KDPCs we really need. + * We must not try setup DPCs for CPUs which aren't there, because that may fail. + */ + RTCPUSET OnlineSet = g_rtMpNtCpuSet; + uint32_t cDpcsNeeded; + switch (enmCpuid) + { + case RT_NT_CPUID_SPECIFIC: + cDpcsNeeded = 1; + break; + case RT_NT_CPUID_PAIR: + cDpcsNeeded = 2; + break; + default: + do + { + cDpcsNeeded = g_cRtMpNtActiveCpus; + OnlineSet = g_rtMpNtCpuSet; + } while (cDpcsNeeded != g_cRtMpNtActiveCpus); + break; + } + + /* + * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs + * and initialize them. + */ + PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC)); + if (!pArgs) + return VERR_NO_MEMORY; + + pArgs->pfnWorker = pfnWorker; + pArgs->pvUser1 = pvUser1; + pArgs->pvUser2 = pvUser2; + pArgs->idCpu = NIL_RTCPUID; + pArgs->idCpu2 = NIL_RTCPUID; + pArgs->cHits = 0; + pArgs->cRefs = 1; + + int rc; + KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1); + if (enmCpuid == RT_NT_CPUID_SPECIFIC) + { + KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs); + if (g_pfnrtKeSetImportanceDpc) + g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance); + rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu); + pArgs->idCpu = idCpu; + } + else if (enmCpuid == RT_NT_CPUID_PAIR) + { + KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs); + if (g_pfnrtKeSetImportanceDpc) + g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance); + rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu); + pArgs->idCpu = idCpu; + + KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs); + if (g_pfnrtKeSetImportanceDpc) + g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance); + if (RT_SUCCESS(rc)) + rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2); + pArgs->idCpu2 = idCpu2; + } + else + { + rc = VINF_SUCCESS; + for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++) + if (RTCpuSetIsMemberByIndex(&OnlineSet, i)) + { + KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs); + if (g_pfnrtKeSetImportanceDpc) + g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance); + rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i)); + } + } + if (RT_FAILURE(rc)) + { + RTMemFree(pArgs); + return rc; + } + + /* + * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu. + * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. + */ + KIRQL oldIrql; + KeRaiseIrql(DISPATCH_LEVEL, &oldIrql); + + /* + * We cannot do other than assume a 1:1 relationship between the + * affinity mask and the process despite the warnings in the docs. + * If someone knows a better way to get this done, please let bird know. + */ + ASMCompilerBarrier(); /* paranoia */ + if (enmCpuid == RT_NT_CPUID_SPECIFIC) + { + ASMAtomicIncS32(&pArgs->cRefs); + BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0); + Assert(fRc); NOREF(fRc); + } + else if (enmCpuid == RT_NT_CPUID_PAIR) + { + ASMAtomicIncS32(&pArgs->cRefs); + BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0); + Assert(fRc); NOREF(fRc); + + ASMAtomicIncS32(&pArgs->cRefs); + fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0); + Assert(fRc); NOREF(fRc); + } + else + { + uint32_t iSelf = RTMpCurSetIndex(); + for (uint32_t i = 0; i < cDpcsNeeded; i++) + { + if ( (i != iSelf) + && RTCpuSetIsMemberByIndex(&OnlineSet, i)) + { + ASMAtomicIncS32(&pArgs->cRefs); + BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0); + Assert(fRc); NOREF(fRc); + } + } + if (enmCpuid != RT_NT_CPUID_OTHERS) + pfnWorker(iSelf, pvUser1, pvUser2); + } + + KeLowerIrql(oldIrql); + + /* + * Flush all DPCs and wait for completion. (can take long!) + */ + /** @todo Consider changing this to an active wait using some atomic inc/dec + * stuff (and check for the current cpu above in the specific case). */ + /** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely + * executed. Seen pArgs being freed while some CPU was using it before + * cRefs was added. */ + if (g_pfnrtNtKeFlushQueuedDpcs) + g_pfnrtNtKeFlushQueuedDpcs(); + + if (pcHits) + *pcHits = pArgs->cHits; + + /* Dereference the argument structure. */ + int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs); + Assert(cRefs >= 0); + if (cRefs == 0) + RTMemFree(pArgs); + + return VINF_SUCCESS; +} + + +RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2) +{ + if (g_pfnrtKeIpiGenericCall) + return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper, + NIL_RTCPUID, NIL_RTCPUID, NULL); + return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL); +} + + +RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2) +{ + if (g_pfnrtKeIpiGenericCall) + return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper, + NIL_RTCPUID, NIL_RTCPUID, NULL); + return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL); +} + + +RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2) +{ + int rc; + AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER); + AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS); + if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall) + return VERR_NOT_SUPPORTED; + + /* + * Check that both CPUs are online before doing the broadcast call. + */ + if ( RTMpIsCpuOnline(idCpu1) + && RTMpIsCpuOnline(idCpu2)) + { + /* + * The broadcast IPI isn't quite as bad as it could have been, because + * it looks like windows doesn't synchronize CPUs on the way out, they + * seems to get back to normal work while the pair is still busy. + */ + uint32_t cHits = 0; + if (g_pfnrtKeIpiGenericCall) + rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits); + else + rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits); + if (RT_SUCCESS(rc)) + { + Assert(cHits <= 2); + if (cHits == 2) + rc = VINF_SUCCESS; + else if (cHits == 1) + rc = VERR_NOT_ALL_CPUS_SHOWED; + else if (cHits == 0) + rc = VERR_CPU_OFFLINE; + else + rc = VERR_CPU_IPE_1; + } + } + /* + * A CPU must be present to be considered just offline. + */ + else if ( RTMpIsCpuPresent(idCpu1) + && RTMpIsCpuPresent(idCpu2)) + rc = VERR_CPU_OFFLINE; + else + rc = VERR_CPU_NOT_FOUND; + return rc; +} + + +RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void) +{ + return g_pfnrtKeIpiGenericCall != NULL; +} + + +/** + * Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it + * when the last reference is released. + */ +DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs) +{ + uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs); + AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs)); + if (cRefs == 0) + RTMemFree(pArgs); +} + + +/** + * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER + * + * @param Dpc DPC object + * @param DeferredContext Context argument specified by KeInitializeDpc + * @param SystemArgument1 Argument specified by KeInsertQueueDpc + * @param SystemArgument2 Argument specified by KeInsertQueueDpc + */ +static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, + IN PVOID SystemArgument1, IN PVOID SystemArgument2) +{ + PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext; + RT_NOREF3(Dpc, SystemArgument1, SystemArgument2); + + ASMAtomicWriteBool(&pArgs->fExecuting, true); + + pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2); + + ASMAtomicWriteBool(&pArgs->fDone, true); + KeSetEvent(&pArgs->DoneEvt, 1 /*PriorityIncrement*/, FALSE /*Wait*/); + + rtMpNtOnSpecificRelease(pArgs); +} + + +RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2) +{ + /* + * Don't try mess with an offline CPU. + */ + if (!RTMpIsCpuOnline(idCpu)) + return !RTMpIsCpuPossible(idCpu) + ? VERR_CPU_NOT_FOUND + : VERR_CPU_OFFLINE; + + /* + * Use the broadcast IPI routine if there are no more than two CPUs online, + * or if the current IRQL is unsuitable for KeWaitForSingleObject. + */ + int rc; + uint32_t cHits = 0; + if ( g_pfnrtKeIpiGenericCall + && ( RTMpGetOnlineCount() <= 2 + || KeGetCurrentIrql() > APC_LEVEL) + ) + { + rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper, + idCpu, NIL_RTCPUID, &cHits); + if (RT_SUCCESS(rc)) + { + if (cHits == 1) + return VINF_SUCCESS; + rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1; + } + return rc; + } + +#if 0 + rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits); + if (RT_SUCCESS(rc)) + { + if (cHits == 1) + return VINF_SUCCESS; + rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1; + } + return rc; + +#else + /* + * Initialize the argument package and the objects within it. + * The package is referenced counted to avoid unnecessary spinning to + * synchronize cleanup and prevent stack corruption. + */ + PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs)); + if (!pArgs) + return VERR_NO_MEMORY; + pArgs->cRefs = 2; + pArgs->fExecuting = false; + pArgs->fDone = false; + pArgs->CallbackArgs.pfnWorker = pfnWorker; + pArgs->CallbackArgs.pvUser1 = pvUser1; + pArgs->CallbackArgs.pvUser2 = pvUser2; + pArgs->CallbackArgs.idCpu = idCpu; + pArgs->CallbackArgs.cHits = 0; + pArgs->CallbackArgs.cRefs = 2; + KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */); + KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs); + if (g_pfnrtKeSetImportanceDpc) + g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance); + rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu); + if (RT_FAILURE(rc)) + { + RTMemFree(pArgs); + return rc; + } + + /* + * Disable preemption while we check the current processor and inserts the DPC. + */ + KIRQL bOldIrql; + KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql); + ASMCompilerBarrier(); /* paranoia */ + + if (RTMpCpuId() == idCpu) + { + /* Just execute the callback on the current CPU. */ + pfnWorker(idCpu, pvUser1, pvUser2); + KeLowerIrql(bOldIrql); + + RTMemFree(pArgs); + return VINF_SUCCESS; + } + + /* Different CPU, so queue it if the CPU is still online. */ + if (RTMpIsCpuOnline(idCpu)) + { + BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0); + Assert(fRc); NOREF(fRc); + KeLowerIrql(bOldIrql); + + uint64_t const nsRealWaitTS = RTTimeNanoTS(); + + /* + * Wait actively for a while in case the CPU/thread responds quickly. + */ + uint32_t cLoopsLeft = 0x20000; + while (cLoopsLeft-- > 0) + { + if (pArgs->fDone) + { + rtMpNtOnSpecificRelease(pArgs); + return VINF_SUCCESS; + } + ASMNopPause(); + } + + /* + * It didn't respond, so wait on the event object, poking the CPU if it's slow. + */ + LARGE_INTEGER Timeout; + Timeout.QuadPart = -10000; /* 1ms */ + NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout); + if (rcNt == STATUS_SUCCESS) + { + rtMpNtOnSpecificRelease(pArgs); + return VINF_SUCCESS; + } + + /* If it hasn't respondend yet, maybe poke it and wait some more. */ + if (rcNt == STATUS_TIMEOUT) + { + if ( !pArgs->fExecuting + && ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus + || g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7)) + RTMpPokeCpu(idCpu); + + Timeout.QuadPart = -1280000; /* 128ms */ + rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout); + if (rcNt == STATUS_SUCCESS) + { + rtMpNtOnSpecificRelease(pArgs); + return VINF_SUCCESS; + } + } + + /* + * Something weird is happening, try bail out. + */ + if (KeRemoveQueueDpc(&pArgs->Dpc)) + { + RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */ + LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt)); + } + else + { + /* DPC is running, wait a good while for it to complete. */ + LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt)); + + Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */ + rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout); + if (rcNt != STATUS_SUCCESS) + LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt)); + } + rc = RTErrConvertFromNtStatus(rcNt); + } + else + { + /* CPU is offline.*/ + KeLowerIrql(bOldIrql); + rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE; + } + + rtMpNtOnSpecificRelease(pArgs); + return rc; +#endif +} + + + + +static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2) +{ + NOREF(Dpc); + NOREF(DeferredContext); + NOREF(SystemArgument1); + NOREF(SystemArgument2); +} + + +/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */ +static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument) +{ + NOREF(Argument); + return 0; +} + + +/** + * RTMpPokeCpu worker that uses broadcast IPIs for doing the work. + * + * @returns VINF_SUCCESS + * @param idCpu The CPU identifier. + */ +int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu) +{ + NOREF(idCpu); + g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0); + return VINF_SUCCESS; +} + + +/** + * RTMpPokeCpu worker that uses the Windows 7 and later version of + * HalRequestIpip to get the job done. + * + * @returns VINF_SUCCESS + * @param idCpu The CPU identifier. + */ +int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu) +{ + /* idCpu is an HAL processor index, so we can use it directly. */ + PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx); + pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */ + g_pfnrtKeInitializeAffinityEx(pTarget); + g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu); + + g_pfnrtHalRequestIpiW7Plus(0, pTarget); + return VINF_SUCCESS; +} + + +/** + * RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip + * to get the job done. + * + * @returns VINF_SUCCESS + * @param idCpu The CPU identifier. + */ +int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu) +{ + __debugbreak(); /** @todo this code needs testing!! */ + KAFFINITY Target = 1; + Target <<= idCpu; + g_pfnrtHalRequestIpiPreW7(Target); + return VINF_SUCCESS; +} + + +int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu) +{ + NOREF(idCpu); + return VERR_NOT_SUPPORTED; +} + + +int rtMpPokeCpuUsingDpc(RTCPUID idCpu) +{ + Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */ + + /* + * APC fallback. + */ + static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {{0}}; + static bool s_fPokeDPCsInitialized = false; + + if (!s_fPokeDPCsInitialized) + { + for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++) + { + KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL); + if (g_pfnrtKeSetImportanceDpc) + g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance); + int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu); + if (RT_FAILURE(rc) && rc != VERR_CPU_NOT_FOUND) + return rc; + } + + s_fPokeDPCsInitialized = true; + } + + /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu. + KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */ + KIRQL oldIrql; + KeRaiseIrql(DISPATCH_LEVEL, &oldIrql); + + if (g_pfnrtKeSetImportanceDpc) + g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance); + g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu); + + /* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately. + Note! Not true on at least Vista & Windows 7 */ + BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0); + + KeLowerIrql(oldIrql); + return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */; +} + + +RTDECL(int) RTMpPokeCpu(RTCPUID idCpu) +{ + if (!RTMpIsCpuOnline(idCpu)) + return !RTMpIsCpuPossible(idCpu) + ? VERR_CPU_NOT_FOUND + : VERR_CPU_OFFLINE; + /* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */ + return g_pfnrtMpPokeCpuWorker(idCpu); +} + + +RTDECL(bool) RTMpOnAllIsConcurrentSafe(void) +{ + return false; +} + |