summaryrefslogtreecommitdiffstats
path: root/src/VBox/VMM/VMMR0/VMMR0.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:17:27 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:17:27 +0000
commitf215e02bf85f68d3a6106c2a1f4f7f063f819064 (patch)
tree6bb5b92c046312c4e95ac2620b10ddf482d3fa8b /src/VBox/VMM/VMMR0/VMMR0.cpp
parentInitial commit. (diff)
downloadvirtualbox-f215e02bf85f68d3a6106c2a1f4f7f063f819064.tar.xz
virtualbox-f215e02bf85f68d3a6106c2a1f4f7f063f819064.zip
Adding upstream version 7.0.14-dfsg.upstream/7.0.14-dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/VBox/VMM/VMMR0/VMMR0.cpp')
-rw-r--r--src/VBox/VMM/VMMR0/VMMR0.cpp3729
1 files changed, 3729 insertions, 0 deletions
diff --git a/src/VBox/VMM/VMMR0/VMMR0.cpp b/src/VBox/VMM/VMMR0/VMMR0.cpp
new file mode 100644
index 00000000..fdec5a0f
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/VMMR0.cpp
@@ -0,0 +1,3729 @@
+/* $Id: VMMR0.cpp $ */
+/** @file
+ * VMM - Host Context Ring 0.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pgm.h>
+#ifdef VBOX_WITH_NEM_R0
+# include <VBox/vmm/nem.h>
+#endif
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/tm.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvm.h>
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+# include <VBox/vmm/pdmpci.h>
+#endif
+#include <VBox/vmm/apic.h>
+
+#include <VBox/vmm/gvmm.h>
+#include <VBox/vmm/gmm.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/intnet.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/version.h>
+#include <VBox/log.h>
+
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#include <iprt/crc.h>
+#include <iprt/initterm.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/mp.h>
+#include <iprt/once.h>
+#include <iprt/semaphore.h>
+#include <iprt/spinlock.h>
+#include <iprt/stdarg.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/timer.h>
+#include <iprt/time.h>
+
+#include "dtrace/VBoxVMM.h"
+
+
+#if defined(_MSC_VER) && defined(RT_ARCH_AMD64) /** @todo check this with with VC7! */
+# pragma intrinsic(_AddressOfReturnAddress)
+#endif
+
+#if defined(RT_OS_DARWIN) && ARCH_BITS == 32
+# error "32-bit darwin is no longer supported. Go back to 4.3 or earlier!"
+#endif
+
+
+/*********************************************************************************************************************************
+* Internal Functions *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD))
+extern uint64_t __udivdi3(uint64_t, uint64_t);
+extern uint64_t __umoddi3(uint64_t, uint64_t);
+#endif
+RT_C_DECLS_END
+static int vmmR0UpdateLoggers(PGVM pGVM, VMCPUID idCpu, PVMMR0UPDATELOGGERSREQ pReq, uint64_t fFlags);
+static int vmmR0LogFlusher(PGVM pGVM);
+static int vmmR0LogWaitFlushed(PGVM pGVM, VMCPUID idCpu, size_t idxLogger);
+static int vmmR0InitLoggers(PGVM pGVM);
+static void vmmR0CleanupLoggers(PGVM pGVM);
+
+
+/*********************************************************************************************************************************
+* Global Variables *
+*********************************************************************************************************************************/
+/** Drag in necessary library bits.
+ * The runtime lives here (in VMMR0.r0) and VBoxDD*R0.r0 links against us. */
+struct CLANG11WEIRDNOTHROW { PFNRT pfn; } g_VMMR0Deps[] =
+{
+ { (PFNRT)RTCrc32 },
+ { (PFNRT)RTOnce },
+#if defined(RT_ARCH_X86) && (defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD))
+ { (PFNRT)__udivdi3 },
+ { (PFNRT)__umoddi3 },
+#endif
+ { NULL }
+};
+
+#ifdef RT_OS_SOLARIS
+/* Dependency information for the native solaris loader. */
+extern "C" { char _depends_on[] = "vboxdrv"; }
+#endif
+
+
+/**
+ * Initialize the module.
+ * This is called when we're first loaded.
+ *
+ * @returns 0 on success.
+ * @returns VBox status on failure.
+ * @param hMod Image handle for use in APIs.
+ */
+DECLEXPORT(int) ModuleInit(void *hMod)
+{
+#ifdef VBOX_WITH_DTRACE_R0
+ /*
+ * The first thing to do is register the static tracepoints.
+ * (Deregistration is automatic.)
+ */
+ int rc2 = SUPR0TracerRegisterModule(hMod, &g_VTGObjHeader);
+ if (RT_FAILURE(rc2))
+ return rc2;
+#endif
+ LogFlow(("ModuleInit:\n"));
+
+#ifdef VBOX_WITH_64ON32_CMOS_DEBUG
+ /*
+ * Display the CMOS debug code.
+ */
+ ASMOutU8(0x72, 0x03);
+ uint8_t bDebugCode = ASMInU8(0x73);
+ LogRel(("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode));
+ RTLogComPrintf("CMOS Debug Code: %#x (%d)\n", bDebugCode, bDebugCode);
+#endif
+
+ /*
+ * Initialize the VMM, GVMM, GMM, HM, PGM (Darwin) and INTNET.
+ */
+ int rc = vmmInitFormatTypes();
+ if (RT_SUCCESS(rc))
+ {
+ rc = GVMMR0Init();
+ if (RT_SUCCESS(rc))
+ {
+ rc = GMMR0Init();
+ if (RT_SUCCESS(rc))
+ {
+ rc = HMR0Init();
+ if (RT_SUCCESS(rc))
+ {
+ PDMR0Init(hMod);
+
+ rc = PGMRegisterStringFormatTypes();
+ if (RT_SUCCESS(rc))
+ {
+ rc = IntNetR0Init();
+ if (RT_SUCCESS(rc))
+ {
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+ rc = PciRawR0Init();
+#endif
+ if (RT_SUCCESS(rc))
+ {
+ rc = CPUMR0ModuleInit();
+ if (RT_SUCCESS(rc))
+ {
+#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+ rc = vmmR0TripleFaultHackInit();
+ if (RT_SUCCESS(rc))
+#endif
+ {
+#ifdef VBOX_WITH_NEM_R0
+ rc = NEMR0Init();
+ if (RT_SUCCESS(rc))
+#endif
+ {
+ LogFlow(("ModuleInit: returns success\n"));
+ return VINF_SUCCESS;
+ }
+ }
+
+ /*
+ * Bail out.
+ */
+#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+ vmmR0TripleFaultHackTerm();
+#endif
+ }
+ else
+ LogRel(("ModuleInit: CPUMR0ModuleInit -> %Rrc\n", rc));
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+ PciRawR0Term();
+#endif
+ }
+ else
+ LogRel(("ModuleInit: PciRawR0Init -> %Rrc\n", rc));
+ IntNetR0Term();
+ }
+ else
+ LogRel(("ModuleInit: IntNetR0Init -> %Rrc\n", rc));
+ PGMDeregisterStringFormatTypes();
+ }
+ else
+ LogRel(("ModuleInit: PGMRegisterStringFormatTypes -> %Rrc\n", rc));
+ HMR0Term();
+ }
+ else
+ LogRel(("ModuleInit: HMR0Init -> %Rrc\n", rc));
+ GMMR0Term();
+ }
+ else
+ LogRel(("ModuleInit: GMMR0Init -> %Rrc\n", rc));
+ GVMMR0Term();
+ }
+ else
+ LogRel(("ModuleInit: GVMMR0Init -> %Rrc\n", rc));
+ vmmTermFormatTypes();
+ }
+ else
+ LogRel(("ModuleInit: vmmInitFormatTypes -> %Rrc\n", rc));
+
+ LogFlow(("ModuleInit: failed %Rrc\n", rc));
+ return rc;
+}
+
+
+/**
+ * Terminate the module.
+ * This is called when we're finally unloaded.
+ *
+ * @param hMod Image handle for use in APIs.
+ */
+DECLEXPORT(void) ModuleTerm(void *hMod)
+{
+ NOREF(hMod);
+ LogFlow(("ModuleTerm:\n"));
+
+ /*
+ * Terminate the CPUM module (Local APIC cleanup).
+ */
+ CPUMR0ModuleTerm();
+
+ /*
+ * Terminate the internal network service.
+ */
+ IntNetR0Term();
+
+ /*
+ * PGM (Darwin), HM and PciRaw global cleanup.
+ */
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+ PciRawR0Term();
+#endif
+ PGMDeregisterStringFormatTypes();
+ HMR0Term();
+#ifdef VBOX_WITH_TRIPLE_FAULT_HACK
+ vmmR0TripleFaultHackTerm();
+#endif
+#ifdef VBOX_WITH_NEM_R0
+ NEMR0Term();
+#endif
+
+ /*
+ * Destroy the GMM and GVMM instances.
+ */
+ GMMR0Term();
+ GVMMR0Term();
+
+ vmmTermFormatTypes();
+ RTTermRunCallbacks(RTTERMREASON_UNLOAD, 0);
+
+ LogFlow(("ModuleTerm: returns\n"));
+}
+
+
+/**
+ * Initializes VMM specific members when the GVM structure is created,
+ * allocating loggers and stuff.
+ *
+ * The loggers are allocated here so that we can update their settings before
+ * doing VMMR0_DO_VMMR0_INIT and have correct logging at that time.
+ *
+ * @returns VBox status code.
+ * @param pGVM The global (ring-0) VM structure.
+ */
+VMMR0_INT_DECL(int) VMMR0InitPerVMData(PGVM pGVM)
+{
+ AssertCompile(sizeof(pGVM->vmmr0.s) <= sizeof(pGVM->vmmr0.padding));
+
+ /*
+ * Initialize all members first.
+ */
+ pGVM->vmmr0.s.fCalledInitVm = false;
+ pGVM->vmmr0.s.hMemObjLogger = NIL_RTR0MEMOBJ;
+ pGVM->vmmr0.s.hMapObjLogger = NIL_RTR0MEMOBJ;
+ pGVM->vmmr0.s.hMemObjReleaseLogger = NIL_RTR0MEMOBJ;
+ pGVM->vmmr0.s.hMapObjReleaseLogger = NIL_RTR0MEMOBJ;
+ pGVM->vmmr0.s.LogFlusher.hSpinlock = NIL_RTSPINLOCK;
+ pGVM->vmmr0.s.LogFlusher.hThread = NIL_RTNATIVETHREAD;
+ pGVM->vmmr0.s.LogFlusher.hEvent = NIL_RTSEMEVENT;
+ pGVM->vmmr0.s.LogFlusher.idxRingHead = 0;
+ pGVM->vmmr0.s.LogFlusher.idxRingTail = 0;
+ pGVM->vmmr0.s.LogFlusher.fThreadWaiting = false;
+
+ for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
+ {
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+ Assert(pGVCpu->idHostCpu == NIL_RTCPUID);
+ Assert(pGVCpu->iHostCpuSet == UINT32_MAX);
+ pGVCpu->vmmr0.s.pPreemptState = NULL;
+ pGVCpu->vmmr0.s.hCtxHook = NIL_RTTHREADCTXHOOK;
+ pGVCpu->vmmr0.s.AssertJmpBuf.pMirrorBuf = &pGVCpu->vmm.s.AssertJmpBuf;
+ pGVCpu->vmmr0.s.AssertJmpBuf.pvStackBuf = &pGVCpu->vmm.s.abAssertStack[0];
+ pGVCpu->vmmr0.s.AssertJmpBuf.cbStackBuf = sizeof(pGVCpu->vmm.s.abAssertStack);
+
+ for (size_t iLogger = 0; iLogger < RT_ELEMENTS(pGVCpu->vmmr0.s.u.aLoggers); iLogger++)
+ pGVCpu->vmmr0.s.u.aLoggers[iLogger].hEventFlushWait = NIL_RTSEMEVENT;
+ }
+
+ /*
+ * Create the loggers.
+ */
+ return vmmR0InitLoggers(pGVM);
+}
+
+
+/**
+ * Initiates the R0 driver for a particular VM instance.
+ *
+ * @returns VBox status code.
+ *
+ * @param pGVM The global (ring-0) VM structure.
+ * @param uSvnRev The SVN revision of the ring-3 part.
+ * @param uBuildType Build type indicator.
+ * @thread EMT(0)
+ */
+static int vmmR0InitVM(PGVM pGVM, uint32_t uSvnRev, uint32_t uBuildType)
+{
+ /*
+ * Match the SVN revisions and build type.
+ */
+ if (uSvnRev != VMMGetSvnRev())
+ {
+ LogRel(("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev()));
+ SUPR0Printf("VMMR0InitVM: Revision mismatch, r3=%d r0=%d\n", uSvnRev, VMMGetSvnRev());
+ return VERR_VMM_R0_VERSION_MISMATCH;
+ }
+ if (uBuildType != vmmGetBuildType())
+ {
+ LogRel(("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType()));
+ SUPR0Printf("VMMR0InitVM: Build type mismatch, r3=%#x r0=%#x\n", uBuildType, vmmGetBuildType());
+ return VERR_VMM_R0_VERSION_MISMATCH;
+ }
+
+ int rc = GVMMR0ValidateGVMandEMT(pGVM, 0 /*idCpu*/);
+ if (RT_FAILURE(rc))
+ return rc;
+
+ /* Don't allow this to be called more than once. */
+ if (!pGVM->vmmr0.s.fCalledInitVm)
+ pGVM->vmmr0.s.fCalledInitVm = true;
+ else
+ return VERR_ALREADY_INITIALIZED;
+
+#ifdef LOG_ENABLED
+
+ /*
+ * Register the EMT R0 logger instance for VCPU 0.
+ */
+ PVMCPUCC pVCpu = VMCC_GET_CPU_0(pGVM);
+ if (pVCpu->vmmr0.s.u.s.Logger.pLogger)
+ {
+# if 0 /* testing of the logger. */
+ LogCom(("vmmR0InitVM: before %p\n", RTLogDefaultInstance()));
+ LogCom(("vmmR0InitVM: pfnFlush=%p actual=%p\n", pR0Logger->Logger.pfnFlush, vmmR0LoggerFlush));
+ LogCom(("vmmR0InitVM: pfnLogger=%p actual=%p\n", pR0Logger->Logger.pfnLogger, vmmR0LoggerWrapper));
+ LogCom(("vmmR0InitVM: offScratch=%d fFlags=%#x fDestFlags=%#x\n", pR0Logger->Logger.offScratch, pR0Logger->Logger.fFlags, pR0Logger->Logger.fDestFlags));
+
+ RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+ LogCom(("vmmR0InitVM: after %p reg\n", RTLogDefaultInstance()));
+ RTLogSetDefaultInstanceThread(NULL, pGVM->pSession);
+ LogCom(("vmmR0InitVM: after %p dereg\n", RTLogDefaultInstance()));
+
+ pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
+ LogCom(("vmmR0InitVM: returned successfully from direct logger call.\n"));
+ pR0Logger->Logger.pfnFlush(&pR0Logger->Logger);
+ LogCom(("vmmR0InitVM: returned successfully from direct flush call.\n"));
+
+ RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+ LogCom(("vmmR0InitVM: after %p reg2\n", RTLogDefaultInstance()));
+ pR0Logger->Logger.pfnLogger("hello ring-0 logger\n");
+ LogCom(("vmmR0InitVM: returned successfully from direct logger call (2). offScratch=%d\n", pR0Logger->Logger.offScratch));
+ RTLogSetDefaultInstanceThread(NULL, pGVM->pSession);
+ LogCom(("vmmR0InitVM: after %p dereg2\n", RTLogDefaultInstance()));
+
+ RTLogLoggerEx(&pR0Logger->Logger, 0, ~0U, "hello ring-0 logger (RTLogLoggerEx)\n");
+ LogCom(("vmmR0InitVM: RTLogLoggerEx returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
+
+ RTLogSetDefaultInstanceThread(&pR0Logger->Logger, (uintptr_t)pGVM->pSession);
+ RTLogPrintf("hello ring-0 logger (RTLogPrintf)\n");
+ LogCom(("vmmR0InitVM: RTLogPrintf returned fine offScratch=%d\n", pR0Logger->Logger.offScratch));
+# endif
+# ifdef VBOX_WITH_R0_LOGGING
+ Log(("Switching to per-thread logging instance %p (key=%p)\n", pVCpu->vmmr0.s.u.s.Logger.pLogger, pGVM->pSession));
+ RTLogSetDefaultInstanceThread(pVCpu->vmmr0.s.u.s.Logger.pLogger, (uintptr_t)pGVM->pSession);
+ pVCpu->vmmr0.s.u.s.Logger.fRegistered = true;
+# endif
+ }
+#endif /* LOG_ENABLED */
+
+ /*
+ * Check if the host supports high resolution timers or not.
+ */
+ if ( pGVM->vmm.s.fUsePeriodicPreemptionTimers
+ && !RTTimerCanDoHighResolution())
+ pGVM->vmm.s.fUsePeriodicPreemptionTimers = false;
+
+ /*
+ * Initialize the per VM data for GVMM and GMM.
+ */
+ rc = GVMMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+ /*
+ * Init HM, CPUM and PGM.
+ */
+ rc = HMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+ rc = CPUMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+ rc = PGMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+ rc = EMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+ rc = IEMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+ rc = IOMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+ rc = PciRawR0InitVM(pGVM);
+#endif
+ if (RT_SUCCESS(rc))
+ {
+ rc = GIMR0InitVM(pGVM);
+ if (RT_SUCCESS(rc))
+ {
+ GVMMR0DoneInitVM(pGVM);
+ PGMR0DoneInitVM(pGVM);
+
+ /*
+ * Collect a bit of info for the VM release log.
+ */
+ pGVM->vmm.s.fIsPreemptPendingApiTrusty = RTThreadPreemptIsPendingTrusty();
+ pGVM->vmm.s.fIsPreemptPossible = RTThreadPreemptIsPossible();;
+ return rc;
+
+ /* bail out*/
+ //GIMR0TermVM(pGVM);
+ }
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+ PciRawR0TermVM(pGVM);
+#endif
+ }
+ }
+ }
+ }
+ }
+ }
+ HMR0TermVM(pGVM);
+ }
+ }
+
+ RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pGVM->pSession);
+ return rc;
+}
+
+
+/**
+ * Does EMT specific VM initialization.
+ *
+ * @returns VBox status code.
+ * @param pGVM The ring-0 VM structure.
+ * @param idCpu The EMT that's calling.
+ */
+static int vmmR0InitVMEmt(PGVM pGVM, VMCPUID idCpu)
+{
+ /* Paranoia (caller checked these already). */
+ AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
+ AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
+
+#if defined(LOG_ENABLED) && defined(VBOX_WITH_R0_LOGGING)
+ /*
+ * Registration of ring 0 loggers.
+ */
+ PVMCPUCC pVCpu = &pGVM->aCpus[idCpu];
+ if ( pVCpu->vmmr0.s.u.s.Logger.pLogger
+ && !pVCpu->vmmr0.s.u.s.Logger.fRegistered)
+ {
+ RTLogSetDefaultInstanceThread(pVCpu->vmmr0.s.u.s.Logger.pLogger, (uintptr_t)pGVM->pSession);
+ pVCpu->vmmr0.s.u.s.Logger.fRegistered = true;
+ }
+#endif
+
+ return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Terminates the R0 bits for a particular VM instance.
+ *
+ * This is normally called by ring-3 as part of the VM termination process, but
+ * may alternatively be called during the support driver session cleanup when
+ * the VM object is destroyed (see GVMM).
+ *
+ * @returns VBox status code.
+ *
+ * @param pGVM The global (ring-0) VM structure.
+ * @param idCpu Set to 0 if EMT(0) or NIL_VMCPUID if session cleanup
+ * thread.
+ * @thread EMT(0) or session clean up thread.
+ */
+VMMR0_INT_DECL(int) VMMR0TermVM(PGVM pGVM, VMCPUID idCpu)
+{
+ /*
+ * Check EMT(0) claim if we're called from userland.
+ */
+ if (idCpu != NIL_VMCPUID)
+ {
+ AssertReturn(idCpu == 0, VERR_INVALID_CPU_ID);
+ int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+ if (RT_FAILURE(rc))
+ return rc;
+ }
+
+#ifdef VBOX_WITH_PCI_PASSTHROUGH
+ PciRawR0TermVM(pGVM);
+#endif
+
+ /*
+ * Tell GVMM what we're up to and check that we only do this once.
+ */
+ if (GVMMR0DoingTermVM(pGVM))
+ {
+ GIMR0TermVM(pGVM);
+
+ /** @todo I wish to call PGMR0PhysFlushHandyPages(pGVM, &pGVM->aCpus[idCpu])
+ * here to make sure we don't leak any shared pages if we crash... */
+ HMR0TermVM(pGVM);
+ }
+
+ /*
+ * Deregister the logger for this EMT.
+ */
+ RTLogSetDefaultInstanceThread(NULL, (uintptr_t)pGVM->pSession);
+
+ /*
+ * Start log flusher thread termination.
+ */
+ ASMAtomicWriteBool(&pGVM->vmmr0.s.LogFlusher.fThreadShutdown, true);
+ if (pGVM->vmmr0.s.LogFlusher.hEvent != NIL_RTSEMEVENT)
+ RTSemEventSignal(pGVM->vmmr0.s.LogFlusher.hEvent);
+
+ return VINF_SUCCESS;
+}
+
+
+/**
+ * This is called at the end of gvmmR0CleanupVM().
+ *
+ * @param pGVM The global (ring-0) VM structure.
+ */
+VMMR0_INT_DECL(void) VMMR0CleanupVM(PGVM pGVM)
+{
+ AssertCompile(NIL_RTTHREADCTXHOOK == (RTTHREADCTXHOOK)0); /* Depends on zero initialized memory working for NIL at the moment. */
+ for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
+ {
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+
+ /** @todo Can we busy wait here for all thread-context hooks to be
+ * deregistered before releasing (destroying) it? Only until we find a
+ * solution for not deregistering hooks everytime we're leaving HMR0
+ * context. */
+ VMMR0ThreadCtxHookDestroyForEmt(pGVCpu);
+ }
+
+ vmmR0CleanupLoggers(pGVM);
+}
+
+
+/**
+ * An interrupt or unhalt force flag is set, deal with it.
+ *
+ * @returns VINF_SUCCESS (or VINF_EM_HALT).
+ * @param pVCpu The cross context virtual CPU structure.
+ * @param uMWait Result from EMMonitorWaitIsActive().
+ * @param enmInterruptibility Guest CPU interruptbility level.
+ */
+static int vmmR0DoHaltInterrupt(PVMCPUCC pVCpu, unsigned uMWait, CPUMINTERRUPTIBILITY enmInterruptibility)
+{
+ Assert(!TRPMHasTrap(pVCpu));
+ Assert( enmInterruptibility > CPUMINTERRUPTIBILITY_INVALID
+ && enmInterruptibility < CPUMINTERRUPTIBILITY_END);
+
+ /*
+ * Pending interrupts w/o any SMIs or NMIs? That the usual case.
+ */
+ if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+ && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI | VMCPU_FF_INTERRUPT_NMI))
+ {
+ if (enmInterruptibility <= CPUMINTERRUPTIBILITY_UNRESTRAINED)
+ {
+ uint8_t u8Interrupt = 0;
+ int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+ Log(("vmmR0DoHaltInterrupt: CPU%d u8Interrupt=%d (%#x) rc=%Rrc\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc));
+ if (RT_SUCCESS(rc))
+ {
+ VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_UNHALT);
+
+ rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
+ AssertRCSuccess(rc);
+ STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
+ return rc;
+ }
+ }
+ }
+ /*
+ * SMI is not implemented yet, at least not here.
+ */
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI))
+ {
+ Log12(("vmmR0DoHaltInterrupt: CPU%d failed #3\n", pVCpu->idCpu));
+ STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+ return VINF_EM_HALT;
+ }
+ /*
+ * NMI.
+ */
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
+ {
+ if (enmInterruptibility < CPUMINTERRUPTIBILITY_NMI_INHIBIT)
+ {
+ /** @todo later. */
+ Log12(("vmmR0DoHaltInterrupt: CPU%d failed #2 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
+ STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+ return VINF_EM_HALT;
+ }
+ }
+ /*
+ * Nested-guest virtual interrupt.
+ */
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
+ {
+ if (enmInterruptibility < CPUMINTERRUPTIBILITY_VIRT_INT_DISABLED)
+ {
+ /** @todo NSTVMX: NSTSVM: Remember, we might have to check and perform VM-exits
+ * here before injecting the virtual interrupt. See emR3ForcedActions
+ * for details. */
+ Log12(("vmmR0DoHaltInterrupt: CPU%d failed #1 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
+ STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+ return VINF_EM_HALT;
+ }
+ }
+
+ if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UNHALT))
+ {
+ STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
+ Log11(("vmmR0DoHaltInterrupt: CPU%d success VINF_SUCCESS (UNHALT)\n", pVCpu->idCpu));
+ return VINF_SUCCESS;
+ }
+ if (uMWait > 1)
+ {
+ STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltExec);
+ Log11(("vmmR0DoHaltInterrupt: CPU%d success VINF_SUCCESS (uMWait=%u > 1)\n", pVCpu->idCpu, uMWait));
+ return VINF_SUCCESS;
+ }
+
+ Log12(("vmmR0DoHaltInterrupt: CPU%d failed #0 (uMWait=%u enmInt=%d)\n", pVCpu->idCpu, uMWait, enmInterruptibility));
+ STAM_REL_COUNTER_INC(&pVCpu->vmm.s.StatR0HaltToR3);
+ return VINF_EM_HALT;
+}
+
+
+/**
+ * This does one round of vmR3HaltGlobal1Halt().
+ *
+ * The rational here is that we'll reduce latency in interrupt situations if we
+ * don't go to ring-3 immediately on a VINF_EM_HALT (guest executed HLT or
+ * MWAIT), but do one round of blocking here instead and hope the interrupt is
+ * raised in the meanwhile.
+ *
+ * If we go to ring-3 we'll quit the inner HM/NEM loop in EM and end up in the
+ * outer loop, which will then call VMR3WaitHalted() and that in turn will do a
+ * ring-0 call (unless we're too close to a timer event). When the interrupt
+ * wakes us up, we'll return from ring-0 and EM will by instinct do a
+ * rescheduling (because of raw-mode) before it resumes the HM/NEM loop and gets
+ * back to VMMR0EntryFast().
+ *
+ * @returns VINF_SUCCESS or VINF_EM_HALT.
+ * @param pGVM The ring-0 VM structure.
+ * @param pGVCpu The ring-0 virtual CPU structure.
+ *
+ * @todo r=bird: All the blocking/waiting and EMT managment should move out of
+ * the VM module, probably to VMM. Then this would be more weird wrt
+ * parameters and statistics.
+ */
+static int vmmR0DoHalt(PGVM pGVM, PGVMCPU pGVCpu)
+{
+ /*
+ * Do spin stat historization.
+ */
+ if (++pGVCpu->vmm.s.cR0Halts & 0xff)
+ { /* likely */ }
+ else if (pGVCpu->vmm.s.cR0HaltsSucceeded > pGVCpu->vmm.s.cR0HaltsToRing3)
+ {
+ pGVCpu->vmm.s.cR0HaltsSucceeded = 2;
+ pGVCpu->vmm.s.cR0HaltsToRing3 = 0;
+ }
+ else
+ {
+ pGVCpu->vmm.s.cR0HaltsSucceeded = 0;
+ pGVCpu->vmm.s.cR0HaltsToRing3 = 2;
+ }
+
+ /*
+ * Flags that makes us go to ring-3.
+ */
+ uint32_t const fVmFFs = VM_FF_TM_VIRTUAL_SYNC | VM_FF_PDM_QUEUES | VM_FF_PDM_DMA
+ | VM_FF_DBGF | VM_FF_REQUEST | VM_FF_CHECK_VM_STATE
+ | VM_FF_RESET | VM_FF_EMT_RENDEZVOUS | VM_FF_PGM_NEED_HANDY_PAGES
+ | VM_FF_PGM_NO_MEMORY | VM_FF_DEBUG_SUSPEND;
+ uint64_t const fCpuFFs = VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT | VMCPU_FF_IEM
+ | VMCPU_FF_REQUEST | VMCPU_FF_DBGF | VMCPU_FF_HM_UPDATE_CR3
+ | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
+ | VMCPU_FF_TO_R3 | VMCPU_FF_IOM;
+
+ /*
+ * Check preconditions.
+ */
+ unsigned const uMWait = EMMonitorWaitIsActive(pGVCpu);
+ CPUMINTERRUPTIBILITY const enmInterruptibility = CPUMGetGuestInterruptibility(pGVCpu);
+ if ( pGVCpu->vmm.s.fMayHaltInRing0
+ && !TRPMHasTrap(pGVCpu)
+ && ( enmInterruptibility == CPUMINTERRUPTIBILITY_UNRESTRAINED
+ || uMWait > 1))
+ {
+ if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+ && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+ {
+ /*
+ * Interrupts pending already?
+ */
+ if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+ APICUpdatePendingInterrupts(pGVCpu);
+
+ /*
+ * Flags that wake up from the halted state.
+ */
+ uint64_t const fIntMask = VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_INTERRUPT_NESTED_GUEST
+ | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI | VMCPU_FF_UNHALT;
+
+ if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+ return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+ ASMNopPause();
+
+ /*
+ * Check out how long till the next timer event.
+ */
+ uint64_t u64Delta;
+ uint64_t u64GipTime = TMTimerPollGIP(pGVM, pGVCpu, &u64Delta);
+
+ if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+ && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+ {
+ if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+ APICUpdatePendingInterrupts(pGVCpu);
+
+ if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+ return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+
+ /*
+ * Wait if there is enough time to the next timer event.
+ */
+ if (u64Delta >= pGVCpu->vmm.s.cNsSpinBlockThreshold)
+ {
+ /* If there are few other CPU cores around, we will procrastinate a
+ little before going to sleep, hoping for some device raising an
+ interrupt or similar. Though, the best thing here would be to
+ dynamically adjust the spin count according to its usfulness or
+ something... */
+ if ( pGVCpu->vmm.s.cR0HaltsSucceeded > pGVCpu->vmm.s.cR0HaltsToRing3
+ && RTMpGetOnlineCount() >= 4)
+ {
+ /** @todo Figure out how we can skip this if it hasn't help recently...
+ * @bugref{9172#c12} */
+ uint32_t cSpinLoops = 42;
+ while (cSpinLoops-- > 0)
+ {
+ ASMNopPause();
+ if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+ APICUpdatePendingInterrupts(pGVCpu);
+ ASMNopPause();
+ if (VM_FF_IS_ANY_SET(pGVM, fVmFFs))
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3FromSpin);
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
+ return VINF_EM_HALT;
+ }
+ ASMNopPause();
+ if (VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3FromSpin);
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
+ return VINF_EM_HALT;
+ }
+ ASMNopPause();
+ if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltExecFromSpin);
+ return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+ }
+ ASMNopPause();
+ }
+ }
+
+ /*
+ * We have to set the state to VMCPUSTATE_STARTED_HALTED here so ring-3
+ * knows when to notify us (cannot access VMINTUSERPERVMCPU::fWait from here).
+ * After changing the state we must recheck the force flags of course.
+ */
+ if (VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED_HALTED, VMCPUSTATE_STARTED))
+ {
+ if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+ && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+ {
+ if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+ APICUpdatePendingInterrupts(pGVCpu);
+
+ if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+ {
+ VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
+ return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+ }
+
+ /* Okay, block! */
+ uint64_t const u64StartSchedHalt = RTTimeNanoTS();
+ int rc = GVMMR0SchedHalt(pGVM, pGVCpu, u64GipTime);
+ uint64_t const u64EndSchedHalt = RTTimeNanoTS();
+ uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
+ Log10(("vmmR0DoHalt: CPU%d: halted %llu ns\n", pGVCpu->idCpu, cNsElapsedSchedHalt));
+
+ VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
+ STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlock, cNsElapsedSchedHalt);
+ if ( rc == VINF_SUCCESS
+ || rc == VERR_INTERRUPTED)
+ {
+ /* Keep some stats like ring-3 does. */
+ int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
+ if (cNsOverslept > 50000)
+ STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockOverslept, cNsOverslept);
+ else if (cNsOverslept < -50000)
+ STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockInsomnia, cNsElapsedSchedHalt);
+ else
+ STAM_REL_PROFILE_ADD_PERIOD(&pGVCpu->vmm.s.StatR0HaltBlockOnTime, cNsElapsedSchedHalt);
+
+ /*
+ * Recheck whether we can resume execution or have to go to ring-3.
+ */
+ if ( !VM_FF_IS_ANY_SET(pGVM, fVmFFs)
+ && !VMCPU_FF_IS_ANY_SET(pGVCpu, fCpuFFs))
+ {
+ if (VMCPU_FF_TEST_AND_CLEAR(pGVCpu, VMCPU_FF_UPDATE_APIC))
+ APICUpdatePendingInterrupts(pGVCpu);
+ if (VMCPU_FF_IS_ANY_SET(pGVCpu, fIntMask))
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltExecFromBlock);
+ return vmmR0DoHaltInterrupt(pGVCpu, uMWait, enmInterruptibility);
+ }
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PostNoInt);
+ Log12(("vmmR0DoHalt: CPU%d post #2 - No pending interrupt\n", pGVCpu->idCpu));
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PostPendingFF);
+ Log12(("vmmR0DoHalt: CPU%d post #1 - Pending FF\n", pGVCpu->idCpu));
+ }
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
+ Log12(("vmmR0DoHalt: CPU%d GVMMR0SchedHalt failed: %Rrc\n", pGVCpu->idCpu, rc));
+ }
+ }
+ else
+ {
+ VMCPU_CMPXCHG_STATE(pGVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_HALTED);
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
+ Log12(("vmmR0DoHalt: CPU%d failed #5 - Pending FF\n", pGVCpu->idCpu));
+ }
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
+ Log12(("vmmR0DoHalt: CPU%d failed #4 - enmState=%d\n", pGVCpu->idCpu, VMCPU_GET_STATE(pGVCpu)));
+ }
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3SmallDelta);
+ Log12(("vmmR0DoHalt: CPU%d failed #3 - delta too small: %RU64\n", pGVCpu->idCpu, u64Delta));
+ }
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
+ Log12(("vmmR0DoHalt: CPU%d failed #2 - Pending FF\n", pGVCpu->idCpu));
+ }
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3PendingFF);
+ Log12(("vmmR0DoHalt: CPU%d failed #1 - Pending FF\n", pGVCpu->idCpu));
+ }
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3Other);
+ Log12(("vmmR0DoHalt: CPU%d failed #0 - fMayHaltInRing0=%d TRPMHasTrap=%d enmInt=%d uMWait=%u\n",
+ pGVCpu->idCpu, pGVCpu->vmm.s.fMayHaltInRing0, TRPMHasTrap(pGVCpu), enmInterruptibility, uMWait));
+ }
+
+ STAM_REL_COUNTER_INC(&pGVCpu->vmm.s.StatR0HaltToR3);
+ return VINF_EM_HALT;
+}
+
+
+/**
+ * VMM ring-0 thread-context callback.
+ *
+ * This does common HM state updating and calls the HM-specific thread-context
+ * callback.
+ *
+ * This is used together with RTThreadCtxHookCreate() on platforms which
+ * supports it, and directly from VMMR0EmtPrepareForBlocking() and
+ * VMMR0EmtResumeAfterBlocking() on platforms which don't.
+ *
+ * @param enmEvent The thread-context event.
+ * @param pvUser Opaque pointer to the VMCPU.
+ *
+ * @thread EMT(pvUser)
+ */
+static DECLCALLBACK(void) vmmR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
+{
+ PVMCPUCC pVCpu = (PVMCPUCC)pvUser;
+
+ switch (enmEvent)
+ {
+ case RTTHREADCTXEVENT_IN:
+ {
+ /*
+ * Linux may call us with preemption enabled (really!) but technically we
+ * cannot get preempted here, otherwise we end up in an infinite recursion
+ * scenario (i.e. preempted in resume hook -> preempt hook -> resume hook...
+ * ad infinitum). Let's just disable preemption for now...
+ */
+ /** @todo r=bird: I don't believe the above. The linux code is clearly enabling
+ * preemption after doing the callout (one or two functions up the
+ * call chain). */
+ /** @todo r=ramshankar: See @bugref{5313#c30}. */
+ RTTHREADPREEMPTSTATE ParanoidPreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+ RTThreadPreemptDisable(&ParanoidPreemptState);
+
+ /* We need to update the VCPU <-> host CPU mapping. */
+ RTCPUID idHostCpu;
+ uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
+ pVCpu->iHostCpuSet = iHostCpuSet;
+ ASMAtomicWriteU32(&pVCpu->idHostCpu, idHostCpu);
+
+ /* In the very unlikely event that the GIP delta for the CPU we're
+ rescheduled needs calculating, try force a return to ring-3.
+ We unfortunately cannot do the measurements right here. */
+ if (RT_LIKELY(!SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
+ { /* likely */ }
+ else
+ VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+
+ /* Invoke the HM-specific thread-context callback. */
+ HMR0ThreadCtxCallback(enmEvent, pvUser);
+
+ /* Restore preemption. */
+ RTThreadPreemptRestore(&ParanoidPreemptState);
+ break;
+ }
+
+ case RTTHREADCTXEVENT_OUT:
+ {
+ /* Invoke the HM-specific thread-context callback. */
+ HMR0ThreadCtxCallback(enmEvent, pvUser);
+
+ /*
+ * Sigh. See VMMGetCpu() used by VMCPU_ASSERT_EMT(). We cannot let several VCPUs
+ * have the same host CPU associated with it.
+ */
+ pVCpu->iHostCpuSet = UINT32_MAX;
+ ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
+ break;
+ }
+
+ default:
+ /* Invoke the HM-specific thread-context callback. */
+ HMR0ThreadCtxCallback(enmEvent, pvUser);
+ break;
+ }
+}
+
+
+/**
+ * Creates thread switching hook for the current EMT thread.
+ *
+ * This is called by GVMMR0CreateVM and GVMMR0RegisterVCpu. If the host
+ * platform does not implement switcher hooks, no hooks will be create and the
+ * member set to NIL_RTTHREADCTXHOOK.
+ *
+ * @returns VBox status code.
+ * @param pVCpu The cross context virtual CPU structure.
+ * @thread EMT(pVCpu)
+ */
+VMMR0_INT_DECL(int) VMMR0ThreadCtxHookCreateForEmt(PVMCPUCC pVCpu)
+{
+ VMCPU_ASSERT_EMT(pVCpu);
+ Assert(pVCpu->vmmr0.s.hCtxHook == NIL_RTTHREADCTXHOOK);
+
+#if 1 /* To disable this stuff change to zero. */
+ int rc = RTThreadCtxHookCreate(&pVCpu->vmmr0.s.hCtxHook, 0, vmmR0ThreadCtxCallback, pVCpu);
+ if (RT_SUCCESS(rc))
+ {
+ pVCpu->pGVM->vmm.s.fIsUsingContextHooks = true;
+ return rc;
+ }
+#else
+ RT_NOREF(vmmR0ThreadCtxCallback);
+ int rc = VERR_NOT_SUPPORTED;
+#endif
+
+ pVCpu->vmmr0.s.hCtxHook = NIL_RTTHREADCTXHOOK;
+ pVCpu->pGVM->vmm.s.fIsUsingContextHooks = false;
+ if (rc == VERR_NOT_SUPPORTED)
+ return VINF_SUCCESS;
+
+ LogRelMax(32, ("RTThreadCtxHookCreate failed! rc=%Rrc pVCpu=%p idCpu=%RU32\n", rc, pVCpu, pVCpu->idCpu));
+ return VINF_SUCCESS; /* Just ignore it, we can live without context hooks. */
+}
+
+
+/**
+ * Destroys the thread switching hook for the specified VCPU.
+ *
+ * @param pVCpu The cross context virtual CPU structure.
+ * @remarks Can be called from any thread.
+ */
+VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDestroyForEmt(PVMCPUCC pVCpu)
+{
+ int rc = RTThreadCtxHookDestroy(pVCpu->vmmr0.s.hCtxHook);
+ AssertRC(rc);
+ pVCpu->vmmr0.s.hCtxHook = NIL_RTTHREADCTXHOOK;
+}
+
+
+/**
+ * Disables the thread switching hook for this VCPU (if we got one).
+ *
+ * @param pVCpu The cross context virtual CPU structure.
+ * @thread EMT(pVCpu)
+ *
+ * @remarks This also clears GVMCPU::idHostCpu, so the mapping is invalid after
+ * this call. This means you have to be careful with what you do!
+ */
+VMMR0_INT_DECL(void) VMMR0ThreadCtxHookDisable(PVMCPUCC pVCpu)
+{
+ /*
+ * Clear the VCPU <-> host CPU mapping as we've left HM context.
+ * @bugref{7726#c19} explains the need for this trick:
+ *
+ * VMXR0CallRing3Callback/SVMR0CallRing3Callback &
+ * hmR0VmxLeaveSession/hmR0SvmLeaveSession disables context hooks during
+ * longjmp & normal return to ring-3, which opens a window where we may be
+ * rescheduled without changing GVMCPUID::idHostCpu and cause confusion if
+ * the CPU starts executing a different EMT. Both functions first disables
+ * preemption and then calls HMR0LeaveCpu which invalids idHostCpu, leaving
+ * an opening for getting preempted.
+ */
+ /** @todo Make HM not need this API! Then we could leave the hooks enabled
+ * all the time. */
+
+ /*
+ * Disable the context hook, if we got one.
+ */
+ if (pVCpu->vmmr0.s.hCtxHook != NIL_RTTHREADCTXHOOK)
+ {
+ Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+ ASMAtomicWriteU32(&pVCpu->idHostCpu, NIL_RTCPUID);
+ int rc = RTThreadCtxHookDisable(pVCpu->vmmr0.s.hCtxHook);
+ AssertRC(rc);
+ }
+}
+
+
+/**
+ * Internal version of VMMR0ThreadCtxHooksAreRegistered.
+ *
+ * @returns true if registered, false otherwise.
+ * @param pVCpu The cross context virtual CPU structure.
+ */
+DECLINLINE(bool) vmmR0ThreadCtxHookIsEnabled(PVMCPUCC pVCpu)
+{
+ return RTThreadCtxHookIsEnabled(pVCpu->vmmr0.s.hCtxHook);
+}
+
+
+/**
+ * Whether thread-context hooks are registered for this VCPU.
+ *
+ * @returns true if registered, false otherwise.
+ * @param pVCpu The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(bool) VMMR0ThreadCtxHookIsEnabled(PVMCPUCC pVCpu)
+{
+ return vmmR0ThreadCtxHookIsEnabled(pVCpu);
+}
+
+
+/**
+ * Returns the ring-0 release logger instance.
+ *
+ * @returns Pointer to release logger, NULL if not configured.
+ * @param pVCpu The cross context virtual CPU structure of the caller.
+ * @thread EMT(pVCpu)
+ */
+VMMR0_INT_DECL(PRTLOGGER) VMMR0GetReleaseLogger(PVMCPUCC pVCpu)
+{
+ return pVCpu->vmmr0.s.u.s.RelLogger.pLogger;
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Record return code statistics
+ * @param pVM The cross context VM structure.
+ * @param pVCpu The cross context virtual CPU structure.
+ * @param rc The status code.
+ */
+static void vmmR0RecordRC(PVMCC pVM, PVMCPUCC pVCpu, int rc)
+{
+ /*
+ * Collect statistics.
+ */
+ switch (rc)
+ {
+ case VINF_SUCCESS:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetNormal);
+ break;
+ case VINF_EM_RAW_INTERRUPT:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterrupt);
+ break;
+ case VINF_EM_RAW_INTERRUPT_HYPER:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptHyper);
+ break;
+ case VINF_EM_RAW_GUEST_TRAP:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGuestTrap);
+ break;
+ case VINF_EM_RAW_RING_SWITCH:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitch);
+ break;
+ case VINF_EM_RAW_RING_SWITCH_INT:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRingSwitchInt);
+ break;
+ case VINF_EM_RAW_STALE_SELECTOR:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetStaleSelector);
+ break;
+ case VINF_EM_RAW_IRET_TRAP:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIRETTrap);
+ break;
+ case VINF_IOM_R3_IOPORT_READ:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIORead);
+ break;
+ case VINF_IOM_R3_IOPORT_WRITE:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOWrite);
+ break;
+ case VINF_IOM_R3_IOPORT_COMMIT_WRITE:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIOCommitWrite);
+ break;
+ case VINF_IOM_R3_MMIO_READ:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIORead);
+ break;
+ case VINF_IOM_R3_MMIO_WRITE:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOWrite);
+ break;
+ case VINF_IOM_R3_MMIO_COMMIT_WRITE:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOCommitWrite);
+ break;
+ case VINF_IOM_R3_MMIO_READ_WRITE:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOReadWrite);
+ break;
+ case VINF_PATM_HC_MMIO_PATCH_READ:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchRead);
+ break;
+ case VINF_PATM_HC_MMIO_PATCH_WRITE:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMMIOPatchWrite);
+ break;
+ case VINF_CPUM_R3_MSR_READ:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRRead);
+ break;
+ case VINF_CPUM_R3_MSR_WRITE:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMSRWrite);
+ break;
+ case VINF_EM_RAW_EMULATE_INSTR:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetEmulate);
+ break;
+ case VINF_PATCH_EMULATE_INSTR:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchEmulate);
+ break;
+ case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetLDTFault);
+ break;
+ case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetGDTFault);
+ break;
+ case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetIDTFault);
+ break;
+ case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTSSFault);
+ break;
+ case VINF_CSAM_PENDING_ACTION:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetCSAMTask);
+ break;
+ case VINF_PGM_SYNC_CR3:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetSyncCR3);
+ break;
+ case VINF_PATM_PATCH_INT3:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchInt3);
+ break;
+ case VINF_PATM_PATCH_TRAP_PF:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchPF);
+ break;
+ case VINF_PATM_PATCH_TRAP_GP:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchGP);
+ break;
+ case VINF_PATM_PENDING_IRQ_AFTER_IRET:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchIretIRQ);
+ break;
+ case VINF_EM_RESCHEDULE_REM:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetRescheduleREM);
+ break;
+ case VINF_EM_RAW_TO_R3:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Total);
+ if (VM_FF_IS_SET(pVM, VM_FF_TM_VIRTUAL_SYNC))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3TMVirt);
+ else if (VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3HandyPages);
+ else if (VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3PDMQueues);
+ else if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Rendezvous);
+ else if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3DMA);
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TIMER))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Timer);
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3CritSect);
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3FF);
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iem);
+ else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IOM))
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Iom);
+ else
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetToR3Unknown);
+ break;
+
+ case VINF_EM_RAW_TIMER_PENDING:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetTimerPending);
+ break;
+ case VINF_EM_RAW_INTERRUPT_PENDING:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetInterruptPending);
+ break;
+ case VINF_PATM_DUPLICATE_FUNCTION:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPATMDuplicateFn);
+ break;
+ case VINF_PGM_POOL_FLUSH_PENDING:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPGMFlushPending);
+ break;
+ case VINF_EM_PENDING_REQUEST:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPendingRequest);
+ break;
+ case VINF_EM_HM_PATCH_TPR_INSTR:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetPatchTPR);
+ break;
+ default:
+ STAM_COUNTER_INC(&pVM->vmm.s.StatRZRetMisc);
+ break;
+ }
+}
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * The Ring 0 entry point, called by the fast-ioctl path.
+ *
+ * @param pGVM The global (ring-0) VM structure.
+ * @param pVMIgnored The cross context VM structure. The return code is
+ * stored in pVM->vmm.s.iLastGZRc.
+ * @param idCpu The Virtual CPU ID of the calling EMT.
+ * @param enmOperation Which operation to execute.
+ * @remarks Assume called with interrupts _enabled_.
+ */
+VMMR0DECL(void) VMMR0EntryFast(PGVM pGVM, PVMCC pVMIgnored, VMCPUID idCpu, VMMR0OPERATION enmOperation)
+{
+ RT_NOREF(pVMIgnored);
+
+ /*
+ * Validation.
+ */
+ if ( idCpu < pGVM->cCpus
+ && pGVM->cCpus == pGVM->cCpusUnsafe)
+ { /*likely*/ }
+ else
+ {
+ SUPR0Printf("VMMR0EntryFast: Bad idCpu=%#x cCpus=%#x cCpusUnsafe=%#x\n", idCpu, pGVM->cCpus, pGVM->cCpusUnsafe);
+ return;
+ }
+
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+ RTNATIVETHREAD const hNativeThread = RTThreadNativeSelf();
+ if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
+ && pGVCpu->hNativeThreadR0 == hNativeThread))
+ { /* likely */ }
+ else
+ {
+ SUPR0Printf("VMMR0EntryFast: Bad thread idCpu=%#x hNativeSelf=%p pGVCpu->hEmt=%p pGVCpu->hNativeThreadR0=%p\n",
+ idCpu, hNativeThread, pGVCpu->hEMT, pGVCpu->hNativeThreadR0);
+ return;
+ }
+
+ /*
+ * Perform requested operation.
+ */
+ switch (enmOperation)
+ {
+ /*
+ * Run guest code using the available hardware acceleration technology.
+ */
+ case VMMR0_DO_HM_RUN:
+ {
+ for (;;) /* hlt loop */
+ {
+ /*
+ * Disable ring-3 calls & blocking till we've successfully entered HM.
+ * Otherwise we sometimes end up blocking at the finall Log4 statement
+ * in VMXR0Enter, while still in a somewhat inbetween state.
+ */
+ VMMRZCallRing3Disable(pGVCpu);
+
+ /*
+ * Disable preemption.
+ */
+ Assert(!vmmR0ThreadCtxHookIsEnabled(pGVCpu));
+ RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+ RTThreadPreemptDisable(&PreemptState);
+ pGVCpu->vmmr0.s.pPreemptState = &PreemptState;
+
+ /*
+ * Get the host CPU identifiers, make sure they are valid and that
+ * we've got a TSC delta for the CPU.
+ */
+ RTCPUID idHostCpu;
+ uint32_t iHostCpuSet = RTMpCurSetIndexAndId(&idHostCpu);
+ if (RT_LIKELY( iHostCpuSet < RTCPUSET_MAX_CPUS
+ && SUPIsTscDeltaAvailableForCpuSetIndex(iHostCpuSet)))
+ {
+ pGVCpu->iHostCpuSet = iHostCpuSet;
+ ASMAtomicWriteU32(&pGVCpu->idHostCpu, idHostCpu);
+
+ /*
+ * Update the periodic preemption timer if it's active.
+ */
+ if (pGVM->vmm.s.fUsePeriodicPreemptionTimers)
+ GVMMR0SchedUpdatePeriodicPreemptionTimer(pGVM, pGVCpu->idHostCpu, TMCalcHostTimerFrequency(pGVM, pGVCpu));
+
+#ifdef VMM_R0_TOUCH_FPU
+ /*
+ * Make sure we've got the FPU state loaded so and we don't need to clear
+ * CR0.TS and get out of sync with the host kernel when loading the guest
+ * FPU state. @ref sec_cpum_fpu (CPUM.cpp) and @bugref{4053}.
+ */
+ CPUMR0TouchHostFpu();
+#endif
+ int rc;
+ bool fPreemptRestored = false;
+ if (!HMR0SuspendPending())
+ {
+ /*
+ * Enable the context switching hook.
+ */
+ if (pGVCpu->vmmr0.s.hCtxHook != NIL_RTTHREADCTXHOOK)
+ {
+ Assert(!RTThreadCtxHookIsEnabled(pGVCpu->vmmr0.s.hCtxHook));
+ int rc2 = RTThreadCtxHookEnable(pGVCpu->vmmr0.s.hCtxHook); AssertRC(rc2);
+ }
+
+ /*
+ * Enter HM context.
+ */
+ rc = HMR0Enter(pGVCpu);
+ if (RT_SUCCESS(rc))
+ {
+ VMCPU_SET_STATE(pGVCpu, VMCPUSTATE_STARTED_HM);
+
+ /*
+ * When preemption hooks are in place, enable preemption now that
+ * we're in HM context.
+ */
+ if (vmmR0ThreadCtxHookIsEnabled(pGVCpu))
+ {
+ fPreemptRestored = true;
+ pGVCpu->vmmr0.s.pPreemptState = NULL;
+ RTThreadPreemptRestore(&PreemptState);
+ }
+ VMMRZCallRing3Enable(pGVCpu);
+
+ /*
+ * Setup the longjmp machinery and execute guest code (calls HMR0RunGuestCode).
+ */
+ rc = vmmR0CallRing3SetJmp(&pGVCpu->vmmr0.s.AssertJmpBuf, HMR0RunGuestCode, pGVM, pGVCpu);
+
+ /*
+ * Assert sanity on the way out. Using manual assertions code here as normal
+ * assertions are going to panic the host since we're outside the setjmp/longjmp zone.
+ */
+ if (RT_UNLIKELY( VMCPU_GET_STATE(pGVCpu) != VMCPUSTATE_STARTED_HM
+ && RT_SUCCESS_NP(rc)
+ && rc != VERR_VMM_RING0_ASSERTION ))
+ {
+ pGVM->vmm.s.szRing0AssertMsg1[0] = '\0';
+ RTStrPrintf(pGVM->vmm.s.szRing0AssertMsg2, sizeof(pGVM->vmm.s.szRing0AssertMsg2),
+ "Got VMCPU state %d expected %d.\n", VMCPU_GET_STATE(pGVCpu), VMCPUSTATE_STARTED_HM);
+ rc = VERR_VMM_WRONG_HM_VMCPU_STATE;
+ }
+#if 0
+ /** @todo Get rid of this. HM shouldn't disable the context hook. */
+ else if (RT_UNLIKELY(vmmR0ThreadCtxHookIsEnabled(pGVCpu)))
+ {
+ pGVM->vmm.s.szRing0AssertMsg1[0] = '\0';
+ RTStrPrintf(pGVM->vmm.s.szRing0AssertMsg2, sizeof(pGVM->vmm.s.szRing0AssertMsg2),
+ "Thread-context hooks still enabled! VCPU=%p Id=%u rc=%d.\n", pGVCpu, pGVCpu->idCpu, rc);
+ rc = VERR_VMM_CONTEXT_HOOK_STILL_ENABLED;
+ }
+#endif
+
+ VMMRZCallRing3Disable(pGVCpu); /* Lazy bird: Simpler just disabling it again... */
+ VMCPU_SET_STATE(pGVCpu, VMCPUSTATE_STARTED);
+ }
+ STAM_COUNTER_INC(&pGVM->vmm.s.StatRunGC);
+
+ /*
+ * Invalidate the host CPU identifiers before we disable the context
+ * hook / restore preemption.
+ */
+ pGVCpu->iHostCpuSet = UINT32_MAX;
+ ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
+
+ /*
+ * Disable context hooks. Due to unresolved cleanup issues, we
+ * cannot leave the hooks enabled when we return to ring-3.
+ *
+ * Note! At the moment HM may also have disabled the hook
+ * when we get here, but the IPRT API handles that.
+ */
+ if (pGVCpu->vmmr0.s.hCtxHook != NIL_RTTHREADCTXHOOK)
+ RTThreadCtxHookDisable(pGVCpu->vmmr0.s.hCtxHook);
+ }
+ /*
+ * The system is about to go into suspend mode; go back to ring 3.
+ */
+ else
+ {
+ pGVCpu->iHostCpuSet = UINT32_MAX;
+ ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
+ rc = VINF_EM_RAW_INTERRUPT;
+ }
+
+ /** @todo When HM stops messing with the context hook state, we'll disable
+ * preemption again before the RTThreadCtxHookDisable call. */
+ if (!fPreemptRestored)
+ {
+ pGVCpu->vmmr0.s.pPreemptState = NULL;
+ RTThreadPreemptRestore(&PreemptState);
+ }
+
+ pGVCpu->vmm.s.iLastGZRc = rc;
+
+ /* Fire dtrace probe and collect statistics. */
+ VBOXVMM_R0_VMM_RETURN_TO_RING3_HM(pGVCpu, CPUMQueryGuestCtxPtr(pGVCpu), rc);
+#ifdef VBOX_WITH_STATISTICS
+ vmmR0RecordRC(pGVM, pGVCpu, rc);
+#endif
+ VMMRZCallRing3Enable(pGVCpu);
+
+ /*
+ * If this is a halt.
+ */
+ if (rc != VINF_EM_HALT)
+ { /* we're not in a hurry for a HLT, so prefer this path */ }
+ else
+ {
+ pGVCpu->vmm.s.iLastGZRc = rc = vmmR0DoHalt(pGVM, pGVCpu);
+ if (rc == VINF_SUCCESS)
+ {
+ pGVCpu->vmm.s.cR0HaltsSucceeded++;
+ continue;
+ }
+ pGVCpu->vmm.s.cR0HaltsToRing3++;
+ }
+ }
+ /*
+ * Invalid CPU set index or TSC delta in need of measuring.
+ */
+ else
+ {
+ pGVCpu->vmmr0.s.pPreemptState = NULL;
+ pGVCpu->iHostCpuSet = UINT32_MAX;
+ ASMAtomicWriteU32(&pGVCpu->idHostCpu, NIL_RTCPUID);
+ RTThreadPreemptRestore(&PreemptState);
+
+ VMMRZCallRing3Enable(pGVCpu);
+
+ if (iHostCpuSet < RTCPUSET_MAX_CPUS)
+ {
+ int rc = SUPR0TscDeltaMeasureBySetIndex(pGVM->pSession, iHostCpuSet, 0 /*fFlags*/,
+ 2 /*cMsWaitRetry*/, 5*RT_MS_1SEC /*cMsWaitThread*/,
+ 0 /*default cTries*/);
+ if (RT_SUCCESS(rc) || rc == VERR_CPU_OFFLINE)
+ pGVCpu->vmm.s.iLastGZRc = VINF_EM_RAW_TO_R3;
+ else
+ pGVCpu->vmm.s.iLastGZRc = rc;
+ }
+ else
+ pGVCpu->vmm.s.iLastGZRc = VERR_INVALID_CPU_INDEX;
+ }
+ break;
+ } /* halt loop. */
+ break;
+ }
+
+#ifdef VBOX_WITH_NEM_R0
+# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
+ case VMMR0_DO_NEM_RUN:
+ {
+ /*
+ * Setup the longjmp machinery and execute guest code (calls NEMR0RunGuestCode).
+ */
+# ifdef VBOXSTRICTRC_STRICT_ENABLED
+ int rc = vmmR0CallRing3SetJmp2(&pGVCpu->vmmr0.s.AssertJmpBuf, (PFNVMMR0SETJMP2)NEMR0RunGuestCode, pGVM, idCpu);
+# else
+ int rc = vmmR0CallRing3SetJmp2(&pGVCpu->vmmr0.s.AssertJmpBuf, NEMR0RunGuestCode, pGVM, idCpu);
+# endif
+ STAM_COUNTER_INC(&pGVM->vmm.s.StatRunGC);
+
+ pGVCpu->vmm.s.iLastGZRc = rc;
+
+ /*
+ * Fire dtrace probe and collect statistics.
+ */
+ VBOXVMM_R0_VMM_RETURN_TO_RING3_NEM(pGVCpu, CPUMQueryGuestCtxPtr(pGVCpu), rc);
+# ifdef VBOX_WITH_STATISTICS
+ vmmR0RecordRC(pGVM, pGVCpu, rc);
+# endif
+ break;
+ }
+# endif
+#endif
+
+ /*
+ * For profiling.
+ */
+ case VMMR0_DO_NOP:
+ pGVCpu->vmm.s.iLastGZRc = VINF_SUCCESS;
+ break;
+
+ /*
+ * Shouldn't happen.
+ */
+ default:
+ AssertMsgFailed(("%#x\n", enmOperation));
+ pGVCpu->vmm.s.iLastGZRc = VERR_NOT_SUPPORTED;
+ break;
+ }
+}
+
+
+/**
+ * Validates a session or VM session argument.
+ *
+ * @returns true / false accordingly.
+ * @param pGVM The global (ring-0) VM structure.
+ * @param pClaimedSession The session claim to validate.
+ * @param pSession The session argument.
+ */
+DECLINLINE(bool) vmmR0IsValidSession(PGVM pGVM, PSUPDRVSESSION pClaimedSession, PSUPDRVSESSION pSession)
+{
+ /* This must be set! */
+ if (!pSession)
+ return false;
+
+ /* Only one out of the two. */
+ if (pGVM && pClaimedSession)
+ return false;
+ if (pGVM)
+ pClaimedSession = pGVM->pSession;
+ return pClaimedSession == pSession;
+}
+
+
+/**
+ * VMMR0EntryEx worker function, either called directly or when ever possible
+ * called thru a longjmp so we can exit safely on failure.
+ *
+ * @returns VBox status code.
+ * @param pGVM The global (ring-0) VM structure.
+ * @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
+ * is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
+ * @param enmOperation Which operation to execute.
+ * @param pReqHdr This points to a SUPVMMR0REQHDR packet. Optional.
+ * The support driver validates this if it's present.
+ * @param u64Arg Some simple constant argument.
+ * @param pSession The session of the caller.
+ *
+ * @remarks Assume called with interrupts _enabled_.
+ */
+DECL_NO_INLINE(static, int) vmmR0EntryExWorker(PGVM pGVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
+ PSUPVMMR0REQHDR pReqHdr, uint64_t u64Arg, PSUPDRVSESSION pSession)
+{
+ /*
+ * Validate pGVM and idCpu for consistency and validity.
+ */
+ if (pGVM != NULL)
+ {
+ if (RT_LIKELY(((uintptr_t)pGVM & HOST_PAGE_OFFSET_MASK) == 0))
+ { /* likely */ }
+ else
+ {
+ SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p! (op=%d)\n", pGVM, enmOperation);
+ return VERR_INVALID_POINTER;
+ }
+
+ if (RT_LIKELY(idCpu == NIL_VMCPUID || idCpu < pGVM->cCpus))
+ { /* likely */ }
+ else
+ {
+ SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu %#x (cCpus=%#x)\n", idCpu, pGVM->cCpus);
+ return VERR_INVALID_PARAMETER;
+ }
+
+ if (RT_LIKELY( pGVM->enmVMState >= VMSTATE_CREATING
+ && pGVM->enmVMState <= VMSTATE_TERMINATED
+ && pGVM->pSession == pSession
+ && pGVM->pSelf == pGVM))
+ { /* likely */ }
+ else
+ {
+ SUPR0Printf("vmmR0EntryExWorker: Invalid pGVM=%p:{.enmVMState=%d, .cCpus=%#x, .pSession=%p(==%p), .pSelf=%p(==%p)}! (op=%d)\n",
+ pGVM, pGVM->enmVMState, pGVM->cCpus, pGVM->pSession, pSession, pGVM->pSelf, pGVM, enmOperation);
+ return VERR_INVALID_POINTER;
+ }
+ }
+ else if (RT_LIKELY(idCpu == NIL_VMCPUID))
+ { /* likely */ }
+ else
+ {
+ SUPR0Printf("vmmR0EntryExWorker: Invalid idCpu=%u\n", idCpu);
+ return VERR_INVALID_PARAMETER;
+ }
+
+ /*
+ * Process the request.
+ */
+ int rc;
+ switch (enmOperation)
+ {
+ /*
+ * GVM requests
+ */
+ case VMMR0_DO_GVMM_CREATE_VM:
+ if (pGVM == NULL && u64Arg == 0 && idCpu == NIL_VMCPUID)
+ rc = GVMMR0CreateVMReq((PGVMMCREATEVMREQ)pReqHdr, pSession);
+ else
+ rc = VERR_INVALID_PARAMETER;
+ break;
+
+ case VMMR0_DO_GVMM_DESTROY_VM:
+ if (pReqHdr == NULL && u64Arg == 0)
+ rc = GVMMR0DestroyVM(pGVM);
+ else
+ rc = VERR_INVALID_PARAMETER;
+ break;
+
+ case VMMR0_DO_GVMM_REGISTER_VMCPU:
+ if (pGVM != NULL)
+ rc = GVMMR0RegisterVCpu(pGVM, idCpu);
+ else
+ rc = VERR_INVALID_PARAMETER;
+ break;
+
+ case VMMR0_DO_GVMM_DEREGISTER_VMCPU:
+ if (pGVM != NULL)
+ rc = GVMMR0DeregisterVCpu(pGVM, idCpu);
+ else
+ rc = VERR_INVALID_PARAMETER;
+ break;
+
+ case VMMR0_DO_GVMM_REGISTER_WORKER_THREAD:
+ if (pGVM != NULL && pReqHdr && pReqHdr->cbReq == sizeof(GVMMREGISTERWORKERTHREADREQ))
+ rc = GVMMR0RegisterWorkerThread(pGVM, (GVMMWORKERTHREAD)(unsigned)u64Arg,
+ ((PGVMMREGISTERWORKERTHREADREQ)(pReqHdr))->hNativeThreadR3);
+ else
+ rc = VERR_INVALID_PARAMETER;
+ break;
+
+ case VMMR0_DO_GVMM_DEREGISTER_WORKER_THREAD:
+ if (pGVM != NULL)
+ rc = GVMMR0DeregisterWorkerThread(pGVM, (GVMMWORKERTHREAD)(unsigned)u64Arg);
+ else
+ rc = VERR_INVALID_PARAMETER;
+ break;
+
+ case VMMR0_DO_GVMM_SCHED_HALT:
+ if (pReqHdr)
+ return VERR_INVALID_PARAMETER;
+ rc = GVMMR0SchedHaltReq(pGVM, idCpu, u64Arg);
+ break;
+
+ case VMMR0_DO_GVMM_SCHED_WAKE_UP:
+ if (pReqHdr || u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GVMMR0SchedWakeUp(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_GVMM_SCHED_POKE:
+ if (pReqHdr || u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GVMMR0SchedPoke(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_GVMM_SCHED_WAKE_UP_AND_POKE_CPUS:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GVMMR0SchedWakeUpAndPokeCpusReq(pGVM, (PGVMMSCHEDWAKEUPANDPOKECPUSREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GVMM_SCHED_POLL:
+ if (pReqHdr || u64Arg > 1)
+ return VERR_INVALID_PARAMETER;
+ rc = GVMMR0SchedPoll(pGVM, idCpu, !!u64Arg);
+ break;
+
+ case VMMR0_DO_GVMM_QUERY_STATISTICS:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GVMMR0QueryStatisticsReq(pGVM, (PGVMMQUERYSTATISTICSSREQ)pReqHdr, pSession);
+ break;
+
+ case VMMR0_DO_GVMM_RESET_STATISTICS:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GVMMR0ResetStatisticsReq(pGVM, (PGVMMRESETSTATISTICSSREQ)pReqHdr, pSession);
+ break;
+
+ /*
+ * Initialize the R0 part of a VM instance.
+ */
+ case VMMR0_DO_VMMR0_INIT:
+ rc = vmmR0InitVM(pGVM, RT_LODWORD(u64Arg), RT_HIDWORD(u64Arg));
+ break;
+
+ /*
+ * Does EMT specific ring-0 init.
+ */
+ case VMMR0_DO_VMMR0_INIT_EMT:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ rc = vmmR0InitVMEmt(pGVM, idCpu);
+ break;
+
+ /*
+ * Terminate the R0 part of a VM instance.
+ */
+ case VMMR0_DO_VMMR0_TERM:
+ rc = VMMR0TermVM(pGVM, 0 /*idCpu*/);
+ break;
+
+ /*
+ * Update release or debug logger instances.
+ */
+ case VMMR0_DO_VMMR0_UPDATE_LOGGERS:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if (!(u64Arg & ~VMMR0UPDATELOGGER_F_VALID_MASK) && pReqHdr != NULL)
+ rc = vmmR0UpdateLoggers(pGVM, idCpu /*idCpu*/, (PVMMR0UPDATELOGGERSREQ)pReqHdr, u64Arg);
+ else
+ return VERR_INVALID_PARAMETER;
+ break;
+
+ /*
+ * Log flusher thread.
+ */
+ case VMMR0_DO_VMMR0_LOG_FLUSHER:
+ if (idCpu != NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if (pReqHdr == NULL && pGVM != NULL)
+ rc = vmmR0LogFlusher(pGVM);
+ else
+ return VERR_INVALID_PARAMETER;
+ break;
+
+ /*
+ * Wait for the flush to finish with all the buffers for the given logger.
+ */
+ case VMMR0_DO_VMMR0_LOG_WAIT_FLUSHED:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if (u64Arg < VMMLOGGER_IDX_MAX && pReqHdr == NULL)
+ rc = vmmR0LogWaitFlushed(pGVM, idCpu /*idCpu*/, (size_t)u64Arg);
+ else
+ return VERR_INVALID_PARAMETER;
+ break;
+
+ /*
+ * Attempt to enable hm mode and check the current setting.
+ */
+ case VMMR0_DO_HM_ENABLE:
+ rc = HMR0EnableAllCpus(pGVM);
+ break;
+
+ /*
+ * Setup the hardware accelerated session.
+ */
+ case VMMR0_DO_HM_SETUP_VM:
+ rc = HMR0SetupVM(pGVM);
+ break;
+
+ /*
+ * PGM wrappers.
+ */
+ case VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ rc = PGMR0PhysAllocateHandyPages(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_PGM_FLUSH_HANDY_PAGES:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ rc = PGMR0PhysFlushHandyPages(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_PGM_ALLOCATE_LARGE_PAGE:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ rc = PGMR0PhysAllocateLargePage(pGVM, idCpu, u64Arg);
+ break;
+
+ case VMMR0_DO_PGM_PHYS_SETUP_IOMMU:
+ if (idCpu != 0)
+ return VERR_INVALID_CPU_ID;
+ rc = PGMR0PhysSetupIoMmu(pGVM);
+ break;
+
+ case VMMR0_DO_PGM_POOL_GROW:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ rc = PGMR0PoolGrow(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_PGM_PHYS_HANDLER_INIT:
+ if (idCpu != 0 || pReqHdr != NULL || u64Arg > UINT32_MAX)
+ return VERR_INVALID_PARAMETER;
+ rc = PGMR0PhysHandlerInitReqHandler(pGVM, (uint32_t)u64Arg);
+ break;
+
+ /*
+ * GMM wrappers.
+ */
+ case VMMR0_DO_GMM_INITIAL_RESERVATION:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0InitialReservationReq(pGVM, idCpu, (PGMMINITIALRESERVATIONREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_UPDATE_RESERVATION:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0UpdateReservationReq(pGVM, idCpu, (PGMMUPDATERESERVATIONREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_ALLOCATE_PAGES:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0AllocatePagesReq(pGVM, idCpu, (PGMMALLOCATEPAGESREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_FREE_PAGES:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0FreePagesReq(pGVM, idCpu, (PGMMFREEPAGESREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_FREE_LARGE_PAGE:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0FreeLargePageReq(pGVM, idCpu, (PGMMFREELARGEPAGEREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0QueryHypervisorMemoryStatsReq((PGMMMEMSTATSREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_QUERY_MEM_STATS:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0QueryMemoryStatsReq(pGVM, idCpu, (PGMMMEMSTATSREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_BALLOONED_PAGES:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0BalloonedPagesReq(pGVM, idCpu, (PGMMBALLOONEDPAGESREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_MAP_UNMAP_CHUNK:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0MapUnmapChunkReq(pGVM, (PGMMMAPUNMAPCHUNKREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_REGISTER_SHARED_MODULE:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0RegisterSharedModuleReq(pGVM, idCpu, (PGMMREGISTERSHAREDMODULEREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0UnregisterSharedModuleReq(pGVM, idCpu, (PGMMUNREGISTERSHAREDMODULEREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_RESET_SHARED_MODULES:
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if ( u64Arg
+ || pReqHdr)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0ResetSharedModules(pGVM, idCpu);
+ break;
+
+#ifdef VBOX_WITH_PAGE_SHARING
+ case VMMR0_DO_GMM_CHECK_SHARED_MODULES:
+ {
+ if (idCpu == NIL_VMCPUID)
+ return VERR_INVALID_CPU_ID;
+ if ( u64Arg
+ || pReqHdr)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0CheckSharedModules(pGVM, idCpu);
+ break;
+ }
+#endif
+
+#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+ case VMMR0_DO_GMM_FIND_DUPLICATE_PAGE:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0FindDuplicatePageReq(pGVM, (PGMMFINDDUPLICATEPAGEREQ)pReqHdr);
+ break;
+#endif
+
+ case VMMR0_DO_GMM_QUERY_STATISTICS:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0QueryStatisticsReq(pGVM, (PGMMQUERYSTATISTICSSREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_GMM_RESET_STATISTICS:
+ if (u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = GMMR0ResetStatisticsReq(pGVM, (PGMMRESETSTATISTICSSREQ)pReqHdr);
+ break;
+
+ /*
+ * A quick GCFGM mock-up.
+ */
+ /** @todo GCFGM with proper access control, ring-3 management interface and all that. */
+ case VMMR0_DO_GCFGM_SET_VALUE:
+ case VMMR0_DO_GCFGM_QUERY_VALUE:
+ {
+ if (pGVM || !pReqHdr || u64Arg || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ PGCFGMVALUEREQ pReq = (PGCFGMVALUEREQ)pReqHdr;
+ if (pReq->Hdr.cbReq != sizeof(*pReq))
+ return VERR_INVALID_PARAMETER;
+ if (enmOperation == VMMR0_DO_GCFGM_SET_VALUE)
+ {
+ rc = GVMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
+ //if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+ // rc = GMMR0SetConfig(pReq->pSession, &pReq->szName[0], pReq->u64Value);
+ }
+ else
+ {
+ rc = GVMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
+ //if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+ // rc = GMMR0QueryConfig(pReq->pSession, &pReq->szName[0], &pReq->u64Value);
+ }
+ break;
+ }
+
+ /*
+ * PDM Wrappers.
+ */
+ case VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER:
+ {
+ if (!pReqHdr || u64Arg || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = PDMR0DriverCallReqHandler(pGVM, (PPDMDRIVERCALLREQHANDLERREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_PDM_DEVICE_CREATE:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = PDMR0DeviceCreateReqHandler(pGVM, (PPDMDEVICECREATEREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_PDM_DEVICE_GEN_CALL:
+ {
+ if (!pReqHdr || u64Arg)
+ return VERR_INVALID_PARAMETER;
+ rc = PDMR0DeviceGenCallReqHandler(pGVM, (PPDMDEVICEGENCALLREQ)pReqHdr, idCpu);
+ break;
+ }
+
+ /** @todo Remove the once all devices has been converted to new style! @bugref{9218} */
+ case VMMR0_DO_PDM_DEVICE_COMPAT_SET_CRITSECT:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = PDMR0DeviceCompatSetCritSectReqHandler(pGVM, (PPDMDEVICECOMPATSETCRITSECTREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_PDM_QUEUE_CREATE:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = PDMR0QueueCreateReqHandler(pGVM, (PPDMQUEUECREATEREQ)pReqHdr);
+ break;
+ }
+
+ /*
+ * Requests to the internal networking service.
+ */
+ case VMMR0_DO_INTNET_OPEN:
+ {
+ PINTNETOPENREQ pReq = (PINTNETOPENREQ)pReqHdr;
+ if (u64Arg || !pReq || !vmmR0IsValidSession(pGVM, pReq->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0OpenReq(pSession, pReq);
+ break;
+ }
+
+ case VMMR0_DO_INTNET_IF_CLOSE:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFCLOSEREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfCloseReq(pSession, (PINTNETIFCLOSEREQ)pReqHdr);
+ break;
+
+
+ case VMMR0_DO_INTNET_IF_GET_BUFFER_PTRS:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFGETBUFFERPTRSREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfGetBufferPtrsReq(pSession, (PINTNETIFGETBUFFERPTRSREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_INTNET_IF_SET_PROMISCUOUS_MODE:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfSetPromiscuousModeReq(pSession, (PINTNETIFSETPROMISCUOUSMODEREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_INTNET_IF_SET_MAC_ADDRESS:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSETMACADDRESSREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfSetMacAddressReq(pSession, (PINTNETIFSETMACADDRESSREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_INTNET_IF_SET_ACTIVE:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSETACTIVEREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfSetActiveReq(pSession, (PINTNETIFSETACTIVEREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_INTNET_IF_SEND:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFSENDREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfSendReq(pSession, (PINTNETIFSENDREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_INTNET_IF_WAIT:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfWaitReq(pSession, (PINTNETIFWAITREQ)pReqHdr);
+ break;
+
+ case VMMR0_DO_INTNET_IF_ABORT_WAIT:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PINTNETIFWAITREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = IntNetR0IfAbortWaitReq(pSession, (PINTNETIFABORTWAITREQ)pReqHdr);
+ break;
+
+#if 0 //def VBOX_WITH_PCI_PASSTHROUGH
+ /*
+ * Requests to host PCI driver service.
+ */
+ case VMMR0_DO_PCIRAW_REQ:
+ if (u64Arg || !pReqHdr || !vmmR0IsValidSession(pGVM, ((PPCIRAWSENDREQ)pReqHdr)->pSession, pSession) || idCpu != NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = PciRawR0ProcessReq(pGVM, pSession, (PPCIRAWSENDREQ)pReqHdr);
+ break;
+#endif
+
+ /*
+ * NEM requests.
+ */
+#ifdef VBOX_WITH_NEM_R0
+# if defined(RT_ARCH_AMD64) && defined(RT_OS_WINDOWS)
+ case VMMR0_DO_NEM_INIT_VM:
+ if (u64Arg || pReqHdr || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0InitVM(pGVM);
+ break;
+
+ case VMMR0_DO_NEM_INIT_VM_PART_2:
+ if (u64Arg || pReqHdr || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0InitVMPart2(pGVM);
+ break;
+
+ case VMMR0_DO_NEM_MAP_PAGES:
+ if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0MapPages(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_NEM_UNMAP_PAGES:
+ if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0UnmapPages(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_NEM_EXPORT_STATE:
+ if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0ExportState(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_NEM_IMPORT_STATE:
+ if (pReqHdr || idCpu == NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0ImportState(pGVM, idCpu, u64Arg);
+ break;
+
+ case VMMR0_DO_NEM_QUERY_CPU_TICK:
+ if (u64Arg || pReqHdr || idCpu == NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0QueryCpuTick(pGVM, idCpu);
+ break;
+
+ case VMMR0_DO_NEM_RESUME_CPU_TICK_ON_ALL:
+ if (pReqHdr || idCpu == NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0ResumeCpuTickOnAll(pGVM, idCpu, u64Arg);
+ break;
+
+ case VMMR0_DO_NEM_UPDATE_STATISTICS:
+ if (u64Arg || pReqHdr)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0UpdateStatistics(pGVM, idCpu);
+ break;
+
+# if 1 && defined(DEBUG_bird)
+ case VMMR0_DO_NEM_EXPERIMENT:
+ if (pReqHdr)
+ return VERR_INVALID_PARAMETER;
+ rc = NEMR0DoExperiment(pGVM, idCpu, u64Arg);
+ break;
+# endif
+# endif
+#endif
+
+ /*
+ * IOM requests.
+ */
+ case VMMR0_DO_IOM_GROW_IO_PORTS:
+ {
+ if (pReqHdr || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = IOMR0IoPortGrowRegistrationTables(pGVM, u64Arg);
+ break;
+ }
+
+ case VMMR0_DO_IOM_GROW_IO_PORT_STATS:
+ {
+ if (pReqHdr || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = IOMR0IoPortGrowStatisticsTable(pGVM, u64Arg);
+ break;
+ }
+
+ case VMMR0_DO_IOM_GROW_MMIO_REGS:
+ {
+ if (pReqHdr || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = IOMR0MmioGrowRegistrationTables(pGVM, u64Arg);
+ break;
+ }
+
+ case VMMR0_DO_IOM_GROW_MMIO_STATS:
+ {
+ if (pReqHdr || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = IOMR0MmioGrowStatisticsTable(pGVM, u64Arg);
+ break;
+ }
+
+ case VMMR0_DO_IOM_SYNC_STATS_INDICES:
+ {
+ if (pReqHdr || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = IOMR0IoPortSyncStatisticsIndices(pGVM);
+ if (RT_SUCCESS(rc))
+ rc = IOMR0MmioSyncStatisticsIndices(pGVM);
+ break;
+ }
+
+ /*
+ * DBGF requests.
+ */
+#ifdef VBOX_WITH_DBGF_TRACING
+ case VMMR0_DO_DBGF_TRACER_CREATE:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = DBGFR0TracerCreateReqHandler(pGVM, (PDBGFTRACERCREATEREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_DBGF_TRACER_CALL_REQ_HANDLER:
+ {
+ if (!pReqHdr || u64Arg)
+ return VERR_INVALID_PARAMETER;
+# if 0 /** @todo */
+ rc = DBGFR0TracerGenCallReqHandler(pGVM, (PDBGFTRACERGENCALLREQ)pReqHdr, idCpu);
+# else
+ rc = VERR_NOT_IMPLEMENTED;
+# endif
+ break;
+ }
+#endif
+
+ case VMMR0_DO_DBGF_BP_INIT:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = DBGFR0BpInitReqHandler(pGVM, (PDBGFBPINITREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_DBGF_BP_CHUNK_ALLOC:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = DBGFR0BpChunkAllocReqHandler(pGVM, (PDBGFBPCHUNKALLOCREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_DBGF_BP_L2_TBL_CHUNK_ALLOC:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = DBGFR0BpL2TblChunkAllocReqHandler(pGVM, (PDBGFBPL2TBLCHUNKALLOCREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_DBGF_BP_OWNER_INIT:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = DBGFR0BpOwnerInitReqHandler(pGVM, (PDBGFBPOWNERINITREQ)pReqHdr);
+ break;
+ }
+
+ case VMMR0_DO_DBGF_BP_PORTIO_INIT:
+ {
+ if (!pReqHdr || u64Arg || idCpu != 0)
+ return VERR_INVALID_PARAMETER;
+ rc = DBGFR0BpPortIoInitReqHandler(pGVM, (PDBGFBPINITREQ)pReqHdr);
+ break;
+ }
+
+
+ /*
+ * TM requests.
+ */
+ case VMMR0_DO_TM_GROW_TIMER_QUEUE:
+ {
+ if (pReqHdr || idCpu == NIL_VMCPUID)
+ return VERR_INVALID_PARAMETER;
+ rc = TMR0TimerQueueGrow(pGVM, RT_HI_U32(u64Arg), RT_LO_U32(u64Arg));
+ break;
+ }
+
+ /*
+ * For profiling.
+ */
+ case VMMR0_DO_NOP:
+ case VMMR0_DO_SLOW_NOP:
+ return VINF_SUCCESS;
+
+ /*
+ * For testing Ring-0 APIs invoked in this environment.
+ */
+ case VMMR0_DO_TESTS:
+ /** @todo make new test */
+ return VINF_SUCCESS;
+
+ default:
+ /*
+ * We're returning VERR_NOT_SUPPORT here so we've got something else
+ * than -1 which the interrupt gate glue code might return.
+ */
+ Log(("operation %#x is not supported\n", enmOperation));
+ return VERR_NOT_SUPPORTED;
+ }
+ return rc;
+}
+
+
+/**
+ * This is just a longjmp wrapper function for VMMR0EntryEx calls.
+ *
+ * @returns VBox status code.
+ * @param pvArgs The argument package
+ */
+static DECLCALLBACK(int) vmmR0EntryExWrapper(void *pvArgs)
+{
+ PGVMCPU pGVCpu = (PGVMCPU)pvArgs;
+ return vmmR0EntryExWorker(pGVCpu->vmmr0.s.pGVM,
+ pGVCpu->vmmr0.s.idCpu,
+ pGVCpu->vmmr0.s.enmOperation,
+ pGVCpu->vmmr0.s.pReq,
+ pGVCpu->vmmr0.s.u64Arg,
+ pGVCpu->vmmr0.s.pSession);
+}
+
+
+/**
+ * The Ring 0 entry point, called by the support library (SUP).
+ *
+ * @returns VBox status code.
+ * @param pGVM The global (ring-0) VM structure.
+ * @param pVM The cross context VM structure.
+ * @param idCpu Virtual CPU ID argument. Must be NIL_VMCPUID if pVM
+ * is NIL_RTR0PTR, and may be NIL_VMCPUID if it isn't
+ * @param enmOperation Which operation to execute.
+ * @param pReq Pointer to the SUPVMMR0REQHDR packet. Optional.
+ * @param u64Arg Some simple constant argument.
+ * @param pSession The session of the caller.
+ * @remarks Assume called with interrupts _enabled_.
+ */
+VMMR0DECL(int) VMMR0EntryEx(PGVM pGVM, PVMCC pVM, VMCPUID idCpu, VMMR0OPERATION enmOperation,
+ PSUPVMMR0REQHDR pReq, uint64_t u64Arg, PSUPDRVSESSION pSession)
+{
+ /*
+ * Requests that should only happen on the EMT thread will be
+ * wrapped in a setjmp so we can assert without causing too much trouble.
+ */
+ if ( pVM != NULL
+ && pGVM != NULL
+ && pVM == pGVM /** @todo drop pVM or pGVM */
+ && idCpu < pGVM->cCpus
+ && pGVM->pSession == pSession
+ && pGVM->pSelf == pGVM
+ && enmOperation != VMMR0_DO_GVMM_DESTROY_VM
+ && enmOperation != VMMR0_DO_GVMM_REGISTER_VMCPU
+ && enmOperation != VMMR0_DO_GVMM_SCHED_WAKE_UP /* idCpu is not caller but target. Sigh. */ /** @todo fix*/
+ && enmOperation != VMMR0_DO_GVMM_SCHED_POKE /* idCpu is not caller but target. Sigh. */ /** @todo fix*/
+ )
+ {
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+ RTNATIVETHREAD hNativeThread = RTThreadNativeSelf();
+ if (RT_LIKELY( pGVCpu->hEMT == hNativeThread
+ && pGVCpu->hNativeThreadR0 == hNativeThread))
+ {
+ pGVCpu->vmmr0.s.pGVM = pGVM;
+ pGVCpu->vmmr0.s.idCpu = idCpu;
+ pGVCpu->vmmr0.s.enmOperation = enmOperation;
+ pGVCpu->vmmr0.s.pReq = pReq;
+ pGVCpu->vmmr0.s.u64Arg = u64Arg;
+ pGVCpu->vmmr0.s.pSession = pSession;
+ return vmmR0CallRing3SetJmpEx(&pGVCpu->vmmr0.s.AssertJmpBuf, vmmR0EntryExWrapper, pGVCpu,
+ ((uintptr_t)u64Arg << 16) | (uintptr_t)enmOperation);
+ }
+ return VERR_VM_THREAD_NOT_EMT;
+ }
+ return vmmR0EntryExWorker(pGVM, idCpu, enmOperation, pReq, u64Arg, pSession);
+}
+
+
+/*********************************************************************************************************************************
+* EMT Blocking *
+*********************************************************************************************************************************/
+
+/**
+ * Checks whether we've armed the ring-0 long jump machinery.
+ *
+ * @returns @c true / @c false
+ * @param pVCpu The cross context virtual CPU structure.
+ * @thread EMT
+ * @sa VMMIsLongJumpArmed
+ */
+VMMR0_INT_DECL(bool) VMMR0IsLongJumpArmed(PVMCPUCC pVCpu)
+{
+#ifdef RT_ARCH_X86
+ return pVCpu->vmmr0.s.AssertJmpBuf.eip != 0;
+#else
+ return pVCpu->vmmr0.s.AssertJmpBuf.rip != 0;
+#endif
+}
+
+
+/**
+ * Locking helper that deals with HM context and checks if the thread can block.
+ *
+ * @returns VINF_SUCCESS if we can block. Returns @a rcBusy or
+ * VERR_VMM_CANNOT_BLOCK if not able to block.
+ * @param pVCpu The cross context virtual CPU structure of the calling
+ * thread.
+ * @param rcBusy What to return in case of a blocking problem. Will IPE
+ * if VINF_SUCCESS and we cannot block.
+ * @param pszCaller The caller (for logging problems).
+ * @param pvLock The lock address (for logging problems).
+ * @param pCtx Where to return context info for the resume call.
+ * @thread EMT(pVCpu)
+ */
+VMMR0_INT_DECL(int) VMMR0EmtPrepareToBlock(PVMCPUCC pVCpu, int rcBusy, const char *pszCaller, void *pvLock,
+ PVMMR0EMTBLOCKCTX pCtx)
+{
+ const char *pszMsg;
+
+ /*
+ * Check that we are allowed to block.
+ */
+ if (RT_LIKELY(VMMRZCallRing3IsEnabled(pVCpu)))
+ {
+ /*
+ * Are we in HM context and w/o a context hook? If so work the context hook.
+ */
+ if (pVCpu->idHostCpu != NIL_RTCPUID)
+ {
+ Assert(pVCpu->iHostCpuSet != UINT32_MAX);
+
+ if (pVCpu->vmmr0.s.hCtxHook == NIL_RTTHREADCTXHOOK)
+ {
+ vmmR0ThreadCtxCallback(RTTHREADCTXEVENT_OUT, pVCpu);
+ if (pVCpu->vmmr0.s.pPreemptState)
+ RTThreadPreemptRestore(pVCpu->vmmr0.s.pPreemptState);
+
+ pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC;
+ pCtx->fWasInHmContext = true;
+ return VINF_SUCCESS;
+ }
+ }
+
+ if (RT_LIKELY(!pVCpu->vmmr0.s.pPreemptState))
+ {
+ /*
+ * Not in HM context or we've got hooks, so just check that preemption
+ * is enabled.
+ */
+ if (RT_LIKELY(RTThreadPreemptIsEnabled(NIL_RTTHREAD)))
+ {
+ pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC;
+ pCtx->fWasInHmContext = false;
+ return VINF_SUCCESS;
+ }
+ pszMsg = "Preemption is disabled!";
+ }
+ else
+ pszMsg = "Preemption state w/o HM state!";
+ }
+ else
+ pszMsg = "Ring-3 calls are disabled!";
+
+ static uint32_t volatile s_cWarnings = 0;
+ if (++s_cWarnings < 50)
+ SUPR0Printf("VMMR0EmtPrepareToBlock: %s pvLock=%p pszCaller=%s rcBusy=%p\n", pszMsg, pvLock, pszCaller, rcBusy);
+ pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC_DEAD;
+ pCtx->fWasInHmContext = false;
+ return rcBusy != VINF_SUCCESS ? rcBusy : VERR_VMM_CANNOT_BLOCK;
+}
+
+
+/**
+ * Counterpart to VMMR0EmtPrepareToBlock.
+ *
+ * @param pVCpu The cross context virtual CPU structure of the calling
+ * thread.
+ * @param pCtx The context structure used with VMMR0EmtPrepareToBlock.
+ * @thread EMT(pVCpu)
+ */
+VMMR0_INT_DECL(void) VMMR0EmtResumeAfterBlocking(PVMCPUCC pVCpu, PVMMR0EMTBLOCKCTX pCtx)
+{
+ AssertReturnVoid(pCtx->uMagic == VMMR0EMTBLOCKCTX_MAGIC);
+ if (pCtx->fWasInHmContext)
+ {
+ if (pVCpu->vmmr0.s.pPreemptState)
+ RTThreadPreemptDisable(pVCpu->vmmr0.s.pPreemptState);
+
+ pCtx->fWasInHmContext = false;
+ vmmR0ThreadCtxCallback(RTTHREADCTXEVENT_IN, pVCpu);
+ }
+ pCtx->uMagic = VMMR0EMTBLOCKCTX_MAGIC_DEAD;
+}
+
+
+/**
+ * Helper for waiting on an RTSEMEVENT, caller did VMMR0EmtPrepareToBlock.
+ *
+ * @returns
+ * @retval VERR_THREAD_IS_TERMINATING
+ * @retval VERR_TIMEOUT if we ended up waiting too long, either according to
+ * @a cMsTimeout or to maximum wait values.
+ *
+ * @param pGVCpu The ring-0 virtual CPU structure.
+ * @param fFlags VMMR0EMTWAIT_F_XXX.
+ * @param hEvent The event to wait on.
+ * @param cMsTimeout The timeout or RT_INDEFINITE_WAIT.
+ */
+VMMR0_INT_DECL(int) VMMR0EmtWaitEventInner(PGVMCPU pGVCpu, uint32_t fFlags, RTSEMEVENT hEvent, RTMSINTERVAL cMsTimeout)
+{
+ AssertReturn(pGVCpu->hEMT == RTThreadNativeSelf(), VERR_VM_THREAD_NOT_EMT);
+
+ /*
+ * Note! Similar code is found in the PDM critical sections too.
+ */
+ uint64_t const nsStart = RTTimeNanoTS();
+ uint64_t cNsMaxTotal = cMsTimeout == RT_INDEFINITE_WAIT
+ ? RT_NS_5MIN : RT_MIN(RT_NS_5MIN, RT_NS_1MS_64 * cMsTimeout);
+ uint32_t cMsMaxOne = RT_MS_5SEC;
+ bool fNonInterruptible = false;
+ for (;;)
+ {
+ /* Wait. */
+ int rcWait = !fNonInterruptible
+ ? RTSemEventWaitNoResume(hEvent, cMsMaxOne)
+ : RTSemEventWait(hEvent, cMsMaxOne);
+ if (RT_SUCCESS(rcWait))
+ return rcWait;
+
+ if (rcWait == VERR_TIMEOUT || rcWait == VERR_INTERRUPTED)
+ {
+ uint64_t const cNsElapsed = RTTimeNanoTS() - nsStart;
+
+ /*
+ * Check the thread termination status.
+ */
+ int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
+ AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING,
+ ("rcTerm=%Rrc\n", rcTerm));
+ if ( rcTerm == VERR_NOT_SUPPORTED
+ && !fNonInterruptible
+ && cNsMaxTotal > RT_NS_1MIN)
+ cNsMaxTotal = RT_NS_1MIN;
+
+ /* We return immediately if it looks like the thread is terminating. */
+ if (rcTerm == VINF_THREAD_IS_TERMINATING)
+ return VERR_THREAD_IS_TERMINATING;
+
+ /* We may suppress VERR_INTERRUPTED if VMMR0EMTWAIT_F_TRY_SUPPRESS_INTERRUPTED was
+ specified, otherwise we'll just return it. */
+ if (rcWait == VERR_INTERRUPTED)
+ {
+ if (!(fFlags & VMMR0EMTWAIT_F_TRY_SUPPRESS_INTERRUPTED))
+ return VERR_INTERRUPTED;
+ if (!fNonInterruptible)
+ {
+ /* First time: Adjust down the wait parameters and make sure we get at least
+ one non-interruptible wait before timing out. */
+ fNonInterruptible = true;
+ cMsMaxOne = 32;
+ uint64_t const cNsLeft = cNsMaxTotal - cNsElapsed;
+ if (cNsLeft > RT_NS_10SEC)
+ cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
+ continue;
+ }
+ }
+
+ /* Check for timeout. */
+ if (cNsElapsed > cNsMaxTotal)
+ return VERR_TIMEOUT;
+ }
+ else
+ return rcWait;
+ }
+ /* not reached */
+}
+
+
+/**
+ * Helper for signalling an SUPSEMEVENT.
+ *
+ * This may temporarily leave the HM context if the host requires that for
+ * signalling SUPSEMEVENT objects.
+ *
+ * @returns VBox status code (see VMMR0EmtPrepareToBlock)
+ * @param pGVM The ring-0 VM structure.
+ * @param pGVCpu The ring-0 virtual CPU structure.
+ * @param hEvent The event to signal.
+ */
+VMMR0_INT_DECL(int) VMMR0EmtSignalSupEvent(PGVM pGVM, PGVMCPU pGVCpu, SUPSEMEVENT hEvent)
+{
+ AssertReturn(pGVCpu->hEMT == RTThreadNativeSelf(), VERR_VM_THREAD_NOT_EMT);
+ if (RTSemEventIsSignalSafe())
+ return SUPSemEventSignal(pGVM->pSession, hEvent);
+
+ VMMR0EMTBLOCKCTX Ctx;
+ int rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, __FUNCTION__, (void *)(uintptr_t)hEvent, &Ctx);
+ if (RT_SUCCESS(rc))
+ {
+ rc = SUPSemEventSignal(pGVM->pSession, hEvent);
+ VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
+ }
+ return rc;
+}
+
+
+/**
+ * Helper for signalling an SUPSEMEVENT, variant supporting non-EMTs.
+ *
+ * This may temporarily leave the HM context if the host requires that for
+ * signalling SUPSEMEVENT objects.
+ *
+ * @returns VBox status code (see VMMR0EmtPrepareToBlock)
+ * @param pGVM The ring-0 VM structure.
+ * @param hEvent The event to signal.
+ */
+VMMR0_INT_DECL(int) VMMR0EmtSignalSupEventByGVM(PGVM pGVM, SUPSEMEVENT hEvent)
+{
+ if (!RTSemEventIsSignalSafe())
+ {
+ PGVMCPU pGVCpu = GVMMR0GetGVCpuByGVMandEMT(pGVM, NIL_RTNATIVETHREAD);
+ if (pGVCpu)
+ {
+ VMMR0EMTBLOCKCTX Ctx;
+ int rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, __FUNCTION__, (void *)(uintptr_t)hEvent, &Ctx);
+ if (RT_SUCCESS(rc))
+ {
+ rc = SUPSemEventSignal(pGVM->pSession, hEvent);
+ VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
+ }
+ return rc;
+ }
+ }
+ return SUPSemEventSignal(pGVM->pSession, hEvent);
+}
+
+
+/*********************************************************************************************************************************
+* Logging. *
+*********************************************************************************************************************************/
+
+/**
+ * VMMR0_DO_VMMR0_UPDATE_LOGGERS: Updates the EMT loggers for the VM.
+ *
+ * @returns VBox status code.
+ * @param pGVM The global (ring-0) VM structure.
+ * @param idCpu The ID of the calling EMT.
+ * @param pReq The request data.
+ * @param fFlags Flags, see VMMR0UPDATELOGGER_F_XXX.
+ * @thread EMT(idCpu)
+ */
+static int vmmR0UpdateLoggers(PGVM pGVM, VMCPUID idCpu, PVMMR0UPDATELOGGERSREQ pReq, uint64_t fFlags)
+{
+ /*
+ * Check sanity. First we require EMT to be calling us.
+ */
+ AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
+ AssertReturn(pGVM->aCpus[idCpu].hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
+
+ AssertReturn(pReq->Hdr.cbReq >= RT_UOFFSETOF_DYN(VMMR0UPDATELOGGERSREQ, afGroups[0]), VERR_INVALID_PARAMETER);
+ AssertReturn(pReq->cGroups < _8K, VERR_INVALID_PARAMETER);
+ AssertReturn(pReq->Hdr.cbReq == RT_UOFFSETOF_DYN(VMMR0UPDATELOGGERSREQ, afGroups[pReq->cGroups]), VERR_INVALID_PARAMETER);
+
+ size_t const idxLogger = (size_t)(fFlags & VMMR0UPDATELOGGER_F_LOGGER_MASK);
+ AssertReturn(idxLogger < VMMLOGGER_IDX_MAX, VERR_OUT_OF_RANGE);
+
+ /*
+ * Adjust flags.
+ */
+ /* Always buffered, unless logging directly to parent VMM: */
+ if (!(fFlags & (VMMR0UPDATELOGGER_F_TO_PARENT_VMM_DBG | VMMR0UPDATELOGGER_F_TO_PARENT_VMM_REL)))
+ pReq->fFlags |= RTLOGFLAGS_BUFFERED;
+ /* These doesn't make sense at present: */
+ pReq->fFlags &= ~(RTLOGFLAGS_FLUSH | RTLOGFLAGS_WRITE_THROUGH);
+ /* We've traditionally skipped the group restrictions. */
+ pReq->fFlags &= ~RTLOGFLAGS_RESTRICT_GROUPS;
+
+ /*
+ * Do the updating.
+ */
+ int rc = VINF_SUCCESS;
+ for (idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
+ {
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+ PRTLOGGER pLogger = pGVCpu->vmmr0.s.u.aLoggers[idxLogger].pLogger;
+ if (pLogger)
+ {
+ pGVCpu->vmmr0.s.u.aLoggers[idxLogger].fFlushToParentVmmDbg = RT_BOOL(fFlags & VMMR0UPDATELOGGER_F_TO_PARENT_VMM_DBG);
+ pGVCpu->vmmr0.s.u.aLoggers[idxLogger].fFlushToParentVmmRel = RT_BOOL(fFlags & VMMR0UPDATELOGGER_F_TO_PARENT_VMM_REL);
+
+ RTLogSetR0ProgramStart(pLogger, pGVM->vmm.s.nsProgramStart);
+ rc = RTLogBulkUpdate(pLogger, pReq->fFlags, pReq->uGroupCrc32, pReq->cGroups, pReq->afGroups);
+ }
+ }
+
+ return rc;
+}
+
+
+/**
+ * VMMR0_DO_VMMR0_LOG_FLUSHER: Get the next log flushing job.
+ *
+ * The job info is copied into VMM::LogFlusherItem.
+ *
+ * @returns VBox status code.
+ * @retval VERR_OBJECT_DESTROYED if we're shutting down.
+ * @retval VERR_NOT_OWNER if the calling thread is not the flusher thread.
+ * @param pGVM The global (ring-0) VM structure.
+ * @thread The log flusher thread (first caller automatically becomes the log
+ * flusher).
+ */
+static int vmmR0LogFlusher(PGVM pGVM)
+{
+ /*
+ * Check that this really is the flusher thread.
+ */
+ RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
+ AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_INTERNAL_ERROR_3);
+ if (RT_LIKELY(pGVM->vmmr0.s.LogFlusher.hThread == hNativeSelf))
+ { /* likely */ }
+ else
+ {
+ /* The first caller becomes the flusher thread. */
+ bool fOk;
+ ASMAtomicCmpXchgHandle(&pGVM->vmmr0.s.LogFlusher.hThread, hNativeSelf, NIL_RTNATIVETHREAD, fOk);
+ if (!fOk)
+ return VERR_NOT_OWNER;
+ pGVM->vmmr0.s.LogFlusher.fThreadRunning = true;
+ }
+
+ /*
+ * Acknowledge flush, waking up waiting EMT.
+ */
+ RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+
+ uint32_t idxTail = pGVM->vmmr0.s.LogFlusher.idxRingTail % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+ uint32_t idxHead = pGVM->vmmr0.s.LogFlusher.idxRingHead % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+ if ( idxTail != idxHead
+ && pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.fProcessing)
+ {
+ /* Pop the head off the ring buffer. */
+ uint32_t const idCpu = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.idCpu;
+ uint32_t const idxLogger = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.idxLogger;
+ uint32_t const idxBuffer = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.idxBuffer;
+
+ pGVM->vmmr0.s.LogFlusher.aRing[idxHead].u32 = UINT32_MAX >> 1; /* invalidate the entry */
+ pGVM->vmmr0.s.LogFlusher.idxRingHead = (idxHead + 1) % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+
+ /* Validate content. */
+ if ( idCpu < pGVM->cCpus
+ && idxLogger < VMMLOGGER_IDX_MAX
+ && idxBuffer < VMMLOGGER_BUFFER_COUNT)
+ {
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+ PVMMR0PERVCPULOGGER pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
+ PVMMR3CPULOGGER pShared = &pGVCpu->vmm.s.u.aLoggers[idxLogger];
+
+ /*
+ * Accounting.
+ */
+ uint32_t cFlushing = pR0Log->cFlushing - 1;
+ if (RT_LIKELY(cFlushing < VMMLOGGER_BUFFER_COUNT))
+ { /*likely*/ }
+ else
+ cFlushing = 0;
+ pR0Log->cFlushing = cFlushing;
+ ASMAtomicWriteU32(&pShared->cFlushing, cFlushing);
+
+ /*
+ * Wake up the EMT if it's waiting.
+ */
+ if (!pR0Log->fEmtWaiting)
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ else
+ {
+ pR0Log->fEmtWaiting = false;
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+
+ int rc = RTSemEventSignal(pR0Log->hEventFlushWait);
+ if (RT_FAILURE(rc))
+ LogRelMax(64, ("vmmR0LogFlusher: RTSemEventSignal failed ACKing entry #%u (%u/%u/%u): %Rrc!\n",
+ idxHead, idCpu, idxLogger, idxBuffer, rc));
+ }
+ }
+ else
+ {
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ LogRelMax(64, ("vmmR0LogFlusher: Bad ACK entry #%u: %u/%u/%u!\n", idxHead, idCpu, idxLogger, idxBuffer));
+ }
+
+ RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ }
+
+ /*
+ * The wait loop.
+ */
+ int rc;
+ for (;;)
+ {
+ /*
+ * Work pending?
+ */
+ idxTail = pGVM->vmmr0.s.LogFlusher.idxRingTail % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+ idxHead = pGVM->vmmr0.s.LogFlusher.idxRingHead % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+ if (idxTail != idxHead)
+ {
+ pGVM->vmmr0.s.LogFlusher.aRing[idxHead].s.fProcessing = true;
+ pGVM->vmm.s.LogFlusherItem.u32 = pGVM->vmmr0.s.LogFlusher.aRing[idxHead].u32;
+
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ return VINF_SUCCESS;
+ }
+
+ /*
+ * Nothing to do, so, check for termination and go to sleep.
+ */
+ if (!pGVM->vmmr0.s.LogFlusher.fThreadShutdown)
+ { /* likely */ }
+ else
+ {
+ rc = VERR_OBJECT_DESTROYED;
+ break;
+ }
+
+ pGVM->vmmr0.s.LogFlusher.fThreadWaiting = true;
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+
+ rc = RTSemEventWaitNoResume(pGVM->vmmr0.s.LogFlusher.hEvent, RT_MS_5MIN);
+
+ RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ pGVM->vmmr0.s.LogFlusher.fThreadWaiting = false;
+
+ if (RT_SUCCESS(rc) || rc == VERR_TIMEOUT)
+ { /* likely */ }
+ else if (rc == VERR_INTERRUPTED)
+ {
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ return rc;
+ }
+ else if (rc == VERR_SEM_DESTROYED || rc == VERR_INVALID_HANDLE)
+ break;
+ else
+ {
+ LogRel(("vmmR0LogFlusher: RTSemEventWaitNoResume returned unexpected status %Rrc\n", rc));
+ break;
+ }
+ }
+
+ /*
+ * Terminating - prevent further calls and indicate to the EMTs that we're no longer around.
+ */
+ pGVM->vmmr0.s.LogFlusher.hThread = ~pGVM->vmmr0.s.LogFlusher.hThread; /* (should be reasonably safe) */
+ pGVM->vmmr0.s.LogFlusher.fThreadRunning = false;
+
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ return rc;
+}
+
+
+/**
+ * VMMR0_DO_VMMR0_LOG_WAIT_FLUSHED: Waits for the flusher thread to finish all
+ * buffers for logger @a idxLogger.
+ *
+ * @returns VBox status code.
+ * @param pGVM The global (ring-0) VM structure.
+ * @param idCpu The ID of the calling EMT.
+ * @param idxLogger Which logger to wait on.
+ * @thread EMT(idCpu)
+ */
+static int vmmR0LogWaitFlushed(PGVM pGVM, VMCPUID idCpu, size_t idxLogger)
+{
+ /*
+ * Check sanity. First we require EMT to be calling us.
+ */
+ AssertReturn(idCpu < pGVM->cCpus, VERR_INVALID_CPU_ID);
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+ AssertReturn(pGVCpu->hEMT == RTThreadNativeSelf(), VERR_INVALID_CPU_ID);
+ AssertReturn(idxLogger < VMMLOGGER_IDX_MAX, VERR_OUT_OF_RANGE);
+ PVMMR0PERVCPULOGGER const pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
+
+ /*
+ * Do the waiting.
+ */
+ int rc = VINF_SUCCESS;
+ RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ uint32_t cFlushing = pR0Log->cFlushing;
+ while (cFlushing > 0)
+ {
+ pR0Log->fEmtWaiting = true;
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+
+ rc = RTSemEventWaitNoResume(pR0Log->hEventFlushWait, RT_MS_5MIN);
+
+ RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ pR0Log->fEmtWaiting = false;
+ if (RT_SUCCESS(rc))
+ {
+ /* Read the new count, make sure it decreased before looping. That
+ way we can guarentee that we will only wait more than 5 min * buffers. */
+ uint32_t const cPrevFlushing = cFlushing;
+ cFlushing = pR0Log->cFlushing;
+ if (cFlushing < cPrevFlushing)
+ continue;
+ rc = VERR_INTERNAL_ERROR_3;
+ }
+ break;
+ }
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ return rc;
+}
+
+
+/**
+ * Inner worker for vmmR0LoggerFlushCommon for flushing to ring-3.
+ */
+static bool vmmR0LoggerFlushInnerToRing3(PGVM pGVM, PGVMCPU pGVCpu, uint32_t idxLogger, size_t idxBuffer, uint32_t cbToFlush)
+{
+ PVMMR0PERVCPULOGGER const pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
+ PVMMR3CPULOGGER const pShared = &pGVCpu->vmm.s.u.aLoggers[idxLogger];
+
+ /*
+ * Figure out what we need to do and whether we can.
+ */
+ enum { kJustSignal, kPrepAndSignal, kPrepSignalAndWait } enmAction;
+#if VMMLOGGER_BUFFER_COUNT >= 2
+ if (pR0Log->cFlushing < VMMLOGGER_BUFFER_COUNT - 1)
+ {
+ if (RTSemEventIsSignalSafe())
+ enmAction = kJustSignal;
+ else if (VMMRZCallRing3IsEnabled(pGVCpu))
+ enmAction = kPrepAndSignal;
+ else
+ {
+ /** @todo This is a bit simplistic. We could introduce a FF to signal the
+ * thread or similar. */
+ STAM_REL_COUNTER_INC(&pShared->StatCannotBlock);
+# if defined(RT_OS_LINUX)
+ SUP_DPRINTF(("vmmR0LoggerFlush: Signalling not safe and EMT blocking disabled! (%u bytes)\n", cbToFlush));
+# endif
+ pShared->cbDropped += cbToFlush;
+ return true;
+ }
+ }
+ else
+#endif
+ if (VMMRZCallRing3IsEnabled(pGVCpu))
+ enmAction = kPrepSignalAndWait;
+ else
+ {
+ STAM_REL_COUNTER_INC(&pShared->StatCannotBlock);
+# if defined(RT_OS_LINUX)
+ SUP_DPRINTF(("vmmR0LoggerFlush: EMT blocking disabled! (%u bytes)\n", cbToFlush));
+# endif
+ pShared->cbDropped += cbToFlush;
+ return true;
+ }
+
+ /*
+ * Prepare for blocking if necessary.
+ */
+ VMMR0EMTBLOCKCTX Ctx;
+ if (enmAction != kJustSignal)
+ {
+ int rc = VMMR0EmtPrepareToBlock(pGVCpu, VINF_SUCCESS, "vmmR0LoggerFlushInnerToRing3", pR0Log->hEventFlushWait, &Ctx);
+ if (RT_SUCCESS(rc))
+ { /* likely */ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pShared->StatCannotBlock);
+ SUP_DPRINTF(("vmmR0LoggerFlush: VMMR0EmtPrepareToBlock failed! rc=%d\n", rc));
+ return false;
+ }
+ }
+
+ /*
+ * Queue the flush job.
+ */
+ bool fFlushedBuffer;
+ RTSpinlockAcquire(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ if (pGVM->vmmr0.s.LogFlusher.fThreadRunning)
+ {
+ uint32_t const idxHead = pGVM->vmmr0.s.LogFlusher.idxRingHead % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+ uint32_t const idxTail = pGVM->vmmr0.s.LogFlusher.idxRingTail % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+ uint32_t const idxNewTail = (idxTail + 1) % RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing);
+ if (idxNewTail != idxHead)
+ {
+ /* Queue it. */
+ pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.idCpu = pGVCpu->idCpu;
+ pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.idxLogger = idxLogger;
+ pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.idxBuffer = (uint32_t)idxBuffer;
+ pGVM->vmmr0.s.LogFlusher.aRing[idxTail].s.fProcessing = 0;
+ pGVM->vmmr0.s.LogFlusher.idxRingTail = idxNewTail;
+
+ /* Update the number of buffers currently being flushed. */
+ uint32_t cFlushing = pR0Log->cFlushing;
+ cFlushing = RT_MIN(cFlushing + 1, VMMLOGGER_BUFFER_COUNT);
+ pShared->cFlushing = pR0Log->cFlushing = cFlushing;
+
+ /* We must wait if all buffers are currently being flushed. */
+ bool const fEmtWaiting = cFlushing >= VMMLOGGER_BUFFER_COUNT && enmAction != kJustSignal /* paranoia */;
+ pR0Log->fEmtWaiting = fEmtWaiting;
+
+ /* Stats. */
+ STAM_REL_COUNTER_INC(&pShared->StatFlushes);
+ STAM_REL_COUNTER_INC(&pGVM->vmm.s.StatLogFlusherFlushes);
+
+ /* Signal the worker thread. */
+ if (pGVM->vmmr0.s.LogFlusher.fThreadWaiting)
+ {
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ RTSemEventSignal(pGVM->vmmr0.s.LogFlusher.hEvent);
+ }
+ else
+ {
+ STAM_REL_COUNTER_INC(&pGVM->vmm.s.StatLogFlusherNoWakeUp);
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ }
+
+ /*
+ * Wait for a buffer to finish flushing.
+ *
+ * Note! Lazy bird is ignoring the status code here. The result is
+ * that we might end up with an extra even signalling and the
+ * next time we need to wait we won't and end up with some log
+ * corruption. However, it's too much hazzle right now for
+ * a scenario which would most likely end the process rather
+ * than causing log corruption.
+ */
+ if (fEmtWaiting)
+ {
+ STAM_REL_PROFILE_START(&pShared->StatWait, a);
+ VMMR0EmtWaitEventInner(pGVCpu, VMMR0EMTWAIT_F_TRY_SUPPRESS_INTERRUPTED,
+ pR0Log->hEventFlushWait, RT_INDEFINITE_WAIT);
+ STAM_REL_PROFILE_STOP(&pShared->StatWait, a);
+ }
+
+ /*
+ * We always switch buffer if we have more than one.
+ */
+#if VMMLOGGER_BUFFER_COUNT == 1
+ fFlushedBuffer = true;
+#else
+ AssertCompile(VMMLOGGER_BUFFER_COUNT >= 1);
+ pShared->idxBuf = (idxBuffer + 1) % VMMLOGGER_BUFFER_COUNT;
+ fFlushedBuffer = false;
+#endif
+ }
+ else
+ {
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ SUP_DPRINTF(("vmmR0LoggerFlush: ring buffer is full!\n"));
+ fFlushedBuffer = true;
+ }
+ }
+ else
+ {
+ RTSpinlockRelease(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ SUP_DPRINTF(("vmmR0LoggerFlush: flusher not active - dropping %u bytes\n", cbToFlush));
+ fFlushedBuffer = true;
+ }
+
+ /*
+ * Restore the HM context.
+ */
+ if (enmAction != kJustSignal)
+ VMMR0EmtResumeAfterBlocking(pGVCpu, &Ctx);
+
+ return fFlushedBuffer;
+}
+
+
+/**
+ * Inner worker for vmmR0LoggerFlushCommon when only flushing to the parent
+ * VMM's logs.
+ */
+static bool vmmR0LoggerFlushInnerToParent(PVMMR0PERVCPULOGGER pR0Log, PRTLOGBUFFERDESC pBufDesc)
+{
+ uint32_t const cbToFlush = pBufDesc->offBuf;
+ if (pR0Log->fFlushToParentVmmDbg)
+ RTLogWriteVmm(pBufDesc->pchBuf, cbToFlush, false /*fRelease*/);
+ if (pR0Log->fFlushToParentVmmRel)
+ RTLogWriteVmm(pBufDesc->pchBuf, cbToFlush, true /*fRelease*/);
+ return true;
+}
+
+
+
+/**
+ * Common worker for vmmR0LogFlush and vmmR0LogRelFlush.
+ */
+static bool vmmR0LoggerFlushCommon(PRTLOGGER pLogger, PRTLOGBUFFERDESC pBufDesc, uint32_t idxLogger)
+{
+ /*
+ * Convert the pLogger into a GVMCPU handle and 'call' back to Ring-3.
+ * (This is a bit paranoid code.)
+ */
+ if (RT_VALID_PTR(pLogger))
+ {
+ if ( pLogger->u32Magic == RTLOGGER_MAGIC
+ && (pLogger->u32UserValue1 & VMMR0_LOGGER_FLAGS_MAGIC_MASK) == VMMR0_LOGGER_FLAGS_MAGIC_VALUE
+ && pLogger->u64UserValue2 == pLogger->u64UserValue3)
+ {
+ PGVMCPU const pGVCpu = (PGVMCPU)(uintptr_t)pLogger->u64UserValue2;
+ if ( RT_VALID_PTR(pGVCpu)
+ && ((uintptr_t)pGVCpu & HOST_PAGE_OFFSET_MASK) == 0)
+ {
+ RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
+ PGVM const pGVM = pGVCpu->pGVM;
+ if ( hNativeSelf == pGVCpu->hEMT
+ && RT_VALID_PTR(pGVM))
+ {
+ PVMMR0PERVCPULOGGER const pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
+ size_t const idxBuffer = pBufDesc - &pR0Log->aBufDescs[0];
+ if (idxBuffer < VMMLOGGER_BUFFER_COUNT)
+ {
+ /*
+ * Make sure we don't recurse forever here should something in the
+ * following code trigger logging or an assertion. Do the rest in
+ * an inner work to avoid hitting the right margin too hard.
+ */
+ if (!pR0Log->fFlushing)
+ {
+ pR0Log->fFlushing = true;
+ bool fFlushed;
+ if ( !pR0Log->fFlushToParentVmmDbg
+ && !pR0Log->fFlushToParentVmmRel)
+ fFlushed = vmmR0LoggerFlushInnerToRing3(pGVM, pGVCpu, idxLogger, idxBuffer, pBufDesc->offBuf);
+ else
+ fFlushed = vmmR0LoggerFlushInnerToParent(pR0Log, pBufDesc);
+ pR0Log->fFlushing = false;
+ return fFlushed;
+ }
+
+ SUP_DPRINTF(("vmmR0LoggerFlush: Recursive flushing!\n"));
+ }
+ else
+ SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p pGVCpu=%p: idxBuffer=%#zx\n", pLogger, pGVCpu, idxBuffer));
+ }
+ else
+ SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p pGVCpu=%p hEMT=%p hNativeSelf=%p!\n",
+ pLogger, pGVCpu, pGVCpu->hEMT, hNativeSelf));
+ }
+ else
+ SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p pGVCpu=%p!\n", pLogger, pGVCpu));
+ }
+ else
+ SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p u32Magic=%#x u32UserValue1=%#x u64UserValue2=%#RX64 u64UserValue3=%#RX64!\n",
+ pLogger, pLogger->u32Magic, pLogger->u32UserValue1, pLogger->u64UserValue2, pLogger->u64UserValue3));
+ }
+ else
+ SUP_DPRINTF(("vmmR0LoggerFlush: pLogger=%p!\n", pLogger));
+ return true;
+}
+
+
+/**
+ * @callback_method_impl{FNRTLOGFLUSH, Release logger buffer flush callback.}
+ */
+static DECLCALLBACK(bool) vmmR0LogRelFlush(PRTLOGGER pLogger, PRTLOGBUFFERDESC pBufDesc)
+{
+ return vmmR0LoggerFlushCommon(pLogger, pBufDesc, VMMLOGGER_IDX_RELEASE);
+}
+
+
+/**
+ * @callback_method_impl{FNRTLOGFLUSH, Logger (debug) buffer flush callback.}
+ */
+static DECLCALLBACK(bool) vmmR0LogFlush(PRTLOGGER pLogger, PRTLOGBUFFERDESC pBufDesc)
+{
+#ifdef LOG_ENABLED
+ return vmmR0LoggerFlushCommon(pLogger, pBufDesc, VMMLOGGER_IDX_REGULAR);
+#else
+ RT_NOREF(pLogger, pBufDesc);
+ return true;
+#endif
+}
+
+
+/*
+ * Override RTLogDefaultInstanceEx so we can do logging from EMTs in ring-0.
+ */
+DECLEXPORT(PRTLOGGER) RTLogDefaultInstanceEx(uint32_t fFlagsAndGroup)
+{
+#ifdef LOG_ENABLED
+ PGVMCPU pGVCpu = GVMMR0GetGVCpuByEMT(NIL_RTNATIVETHREAD);
+ if (pGVCpu)
+ {
+ PRTLOGGER pLogger = pGVCpu->vmmr0.s.u.s.Logger.pLogger;
+ if (RT_VALID_PTR(pLogger))
+ {
+ if ( pLogger->u64UserValue2 == (uintptr_t)pGVCpu
+ && pLogger->u64UserValue3 == (uintptr_t)pGVCpu)
+ {
+ if (!pGVCpu->vmmr0.s.u.s.Logger.fFlushing)
+ return RTLogCheckGroupFlags(pLogger, fFlagsAndGroup);
+
+ /*
+ * When we're flushing we _must_ return NULL here to suppress any
+ * attempts at using the logger while in vmmR0LoggerFlushCommon.
+ * The VMMR0EmtPrepareToBlock code may trigger logging in HM,
+ * which will reset the buffer content before we even get to queue
+ * the flush request. (Only an issue when VBOX_WITH_R0_LOGGING
+ * is enabled.)
+ */
+ return NULL;
+ }
+ }
+ }
+#endif
+ return SUPR0DefaultLogInstanceEx(fFlagsAndGroup);
+}
+
+
+/*
+ * Override RTLogRelGetDefaultInstanceEx so we can do LogRel to VBox.log from EMTs in ring-0.
+ */
+DECLEXPORT(PRTLOGGER) RTLogRelGetDefaultInstanceEx(uint32_t fFlagsAndGroup)
+{
+ PGVMCPU pGVCpu = GVMMR0GetGVCpuByEMT(NIL_RTNATIVETHREAD);
+ if (pGVCpu)
+ {
+ PRTLOGGER pLogger = pGVCpu->vmmr0.s.u.s.RelLogger.pLogger;
+ if (RT_VALID_PTR(pLogger))
+ {
+ if ( pLogger->u64UserValue2 == (uintptr_t)pGVCpu
+ && pLogger->u64UserValue3 == (uintptr_t)pGVCpu)
+ {
+ if (!pGVCpu->vmmr0.s.u.s.RelLogger.fFlushing)
+ return RTLogCheckGroupFlags(pLogger, fFlagsAndGroup);
+
+ /* ASSUMES no LogRels hidden within the VMMR0EmtPrepareToBlock code
+ path, so we don't return NULL here like for the debug logger... */
+ }
+ }
+ }
+ return SUPR0GetDefaultLogRelInstanceEx(fFlagsAndGroup);
+}
+
+
+/**
+ * Helper for vmmR0InitLoggerSet
+ */
+static int vmmR0InitLoggerOne(PGVMCPU pGVCpu, bool fRelease, PVMMR0PERVCPULOGGER pR0Log, PVMMR3CPULOGGER pShared,
+ uint32_t cbBuf, char *pchBuf, RTR3PTR pchBufR3)
+{
+ /*
+ * Create and configure the logger.
+ */
+ for (size_t i = 0; i < VMMLOGGER_BUFFER_COUNT; i++)
+ {
+ pR0Log->aBufDescs[i].u32Magic = RTLOGBUFFERDESC_MAGIC;
+ pR0Log->aBufDescs[i].uReserved = 0;
+ pR0Log->aBufDescs[i].cbBuf = cbBuf;
+ pR0Log->aBufDescs[i].offBuf = 0;
+ pR0Log->aBufDescs[i].pchBuf = pchBuf + i * cbBuf;
+ pR0Log->aBufDescs[i].pAux = &pShared->aBufs[i].AuxDesc;
+
+ pShared->aBufs[i].AuxDesc.fFlushedIndicator = false;
+ pShared->aBufs[i].AuxDesc.afPadding[0] = 0;
+ pShared->aBufs[i].AuxDesc.afPadding[1] = 0;
+ pShared->aBufs[i].AuxDesc.afPadding[2] = 0;
+ pShared->aBufs[i].AuxDesc.offBuf = 0;
+ pShared->aBufs[i].pchBufR3 = pchBufR3 + i * cbBuf;
+ }
+ pShared->cbBuf = cbBuf;
+
+ static const char * const s_apszGroups[] = VBOX_LOGGROUP_NAMES;
+ int rc = RTLogCreateEx(&pR0Log->pLogger, fRelease ? "VBOX_RELEASE_LOG" : "VBOX_LOG", RTLOG_F_NO_LOCKING | RTLOGFLAGS_BUFFERED,
+ "all", RT_ELEMENTS(s_apszGroups), s_apszGroups, UINT32_MAX,
+ VMMLOGGER_BUFFER_COUNT, pR0Log->aBufDescs, RTLOGDEST_DUMMY,
+ NULL /*pfnPhase*/, 0 /*cHistory*/, 0 /*cbHistoryFileMax*/, 0 /*cSecsHistoryTimeSlot*/,
+ NULL /*pOutputIf*/, NULL /*pvOutputIfUser*/,
+ NULL /*pErrInfo*/, NULL /*pszFilenameFmt*/);
+ if (RT_SUCCESS(rc))
+ {
+ PRTLOGGER pLogger = pR0Log->pLogger;
+ pLogger->u32UserValue1 = VMMR0_LOGGER_FLAGS_MAGIC_VALUE;
+ pLogger->u64UserValue2 = (uintptr_t)pGVCpu;
+ pLogger->u64UserValue3 = (uintptr_t)pGVCpu;
+
+ rc = RTLogSetFlushCallback(pLogger, fRelease ? vmmR0LogRelFlush : vmmR0LogFlush);
+ if (RT_SUCCESS(rc))
+ {
+ RTLogSetR0ThreadNameF(pLogger, "EMT-%u-R0", pGVCpu->idCpu);
+
+ /*
+ * Create the event sem the EMT waits on while flushing is happening.
+ */
+ rc = RTSemEventCreate(&pR0Log->hEventFlushWait);
+ if (RT_SUCCESS(rc))
+ return VINF_SUCCESS;
+ pR0Log->hEventFlushWait = NIL_RTSEMEVENT;
+ }
+ RTLogDestroy(pLogger);
+ }
+ pR0Log->pLogger = NULL;
+ return rc;
+}
+
+
+/**
+ * Worker for VMMR0CleanupVM and vmmR0InitLoggerSet that destroys one logger.
+ */
+static void vmmR0TermLoggerOne(PVMMR0PERVCPULOGGER pR0Log, PVMMR3CPULOGGER pShared)
+{
+ RTLogDestroy(pR0Log->pLogger);
+ pR0Log->pLogger = NULL;
+
+ for (size_t i = 0; i < VMMLOGGER_BUFFER_COUNT; i++)
+ pShared->aBufs[i].pchBufR3 = NIL_RTR3PTR;
+
+ RTSemEventDestroy(pR0Log->hEventFlushWait);
+ pR0Log->hEventFlushWait = NIL_RTSEMEVENT;
+}
+
+
+/**
+ * Initializes one type of loggers for each EMT.
+ */
+static int vmmR0InitLoggerSet(PGVM pGVM, uint8_t idxLogger, uint32_t cbBuf, PRTR0MEMOBJ phMemObj, PRTR0MEMOBJ phMapObj)
+{
+ /* Allocate buffers first. */
+ int rc = RTR0MemObjAllocPage(phMemObj, cbBuf * pGVM->cCpus * VMMLOGGER_BUFFER_COUNT, false /*fExecutable*/);
+ if (RT_SUCCESS(rc))
+ {
+ rc = RTR0MemObjMapUser(phMapObj, *phMemObj, (RTR3PTR)-1, 0 /*uAlignment*/, RTMEM_PROT_READ, NIL_RTR0PROCESS);
+ if (RT_SUCCESS(rc))
+ {
+ char * const pchBuf = (char *)RTR0MemObjAddress(*phMemObj);
+ AssertPtrReturn(pchBuf, VERR_INTERNAL_ERROR_2);
+
+ RTR3PTR const pchBufR3 = RTR0MemObjAddressR3(*phMapObj);
+ AssertReturn(pchBufR3 != NIL_RTR3PTR, VERR_INTERNAL_ERROR_3);
+
+ /* Initialize the per-CPU loggers. */
+ for (uint32_t i = 0; i < pGVM->cCpus; i++)
+ {
+ PGVMCPU pGVCpu = &pGVM->aCpus[i];
+ PVMMR0PERVCPULOGGER pR0Log = &pGVCpu->vmmr0.s.u.aLoggers[idxLogger];
+ PVMMR3CPULOGGER pShared = &pGVCpu->vmm.s.u.aLoggers[idxLogger];
+ rc = vmmR0InitLoggerOne(pGVCpu, idxLogger == VMMLOGGER_IDX_RELEASE, pR0Log, pShared, cbBuf,
+ pchBuf + i * cbBuf * VMMLOGGER_BUFFER_COUNT,
+ pchBufR3 + i * cbBuf * VMMLOGGER_BUFFER_COUNT);
+ if (RT_FAILURE(rc))
+ {
+ vmmR0TermLoggerOne(pR0Log, pShared);
+ while (i-- > 0)
+ {
+ pGVCpu = &pGVM->aCpus[i];
+ vmmR0TermLoggerOne(&pGVCpu->vmmr0.s.u.aLoggers[idxLogger], &pGVCpu->vmm.s.u.aLoggers[idxLogger]);
+ }
+ break;
+ }
+ }
+ if (RT_SUCCESS(rc))
+ return VINF_SUCCESS;
+
+ /* Bail out. */
+ RTR0MemObjFree(*phMapObj, false /*fFreeMappings*/);
+ *phMapObj = NIL_RTR0MEMOBJ;
+ }
+ RTR0MemObjFree(*phMemObj, true /*fFreeMappings*/);
+ *phMemObj = NIL_RTR0MEMOBJ;
+ }
+ return rc;
+}
+
+
+/**
+ * Worker for VMMR0InitPerVMData that initializes all the logging related stuff.
+ *
+ * @returns VBox status code.
+ * @param pGVM The global (ring-0) VM structure.
+ */
+static int vmmR0InitLoggers(PGVM pGVM)
+{
+ /*
+ * Invalidate the ring buffer (not really necessary).
+ */
+ for (size_t idx = 0; idx < RT_ELEMENTS(pGVM->vmmr0.s.LogFlusher.aRing); idx++)
+ pGVM->vmmr0.s.LogFlusher.aRing[idx].u32 = UINT32_MAX >> 1; /* (all bits except fProcessing set) */
+
+ /*
+ * Create the spinlock and flusher event semaphore.
+ */
+ int rc = RTSpinlockCreate(&pGVM->vmmr0.s.LogFlusher.hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "VM-Log-Flusher");
+ if (RT_SUCCESS(rc))
+ {
+ rc = RTSemEventCreate(&pGVM->vmmr0.s.LogFlusher.hEvent);
+ if (RT_SUCCESS(rc))
+ {
+ /*
+ * Create the ring-0 release loggers.
+ */
+ rc = vmmR0InitLoggerSet(pGVM, VMMLOGGER_IDX_RELEASE, _4K,
+ &pGVM->vmmr0.s.hMemObjReleaseLogger, &pGVM->vmmr0.s.hMapObjReleaseLogger);
+#ifdef LOG_ENABLED
+ if (RT_SUCCESS(rc))
+ {
+ /*
+ * Create debug loggers.
+ */
+ rc = vmmR0InitLoggerSet(pGVM, VMMLOGGER_IDX_REGULAR, _64K,
+ &pGVM->vmmr0.s.hMemObjLogger, &pGVM->vmmr0.s.hMapObjLogger);
+ }
+#endif
+ }
+ }
+ return rc;
+}
+
+
+/**
+ * Worker for VMMR0InitPerVMData that initializes all the logging related stuff.
+ *
+ * @param pGVM The global (ring-0) VM structure.
+ */
+static void vmmR0CleanupLoggers(PGVM pGVM)
+{
+ for (VMCPUID idCpu = 0; idCpu < pGVM->cCpus; idCpu++)
+ {
+ PGVMCPU pGVCpu = &pGVM->aCpus[idCpu];
+ for (size_t iLogger = 0; iLogger < RT_ELEMENTS(pGVCpu->vmmr0.s.u.aLoggers); iLogger++)
+ vmmR0TermLoggerOne(&pGVCpu->vmmr0.s.u.aLoggers[iLogger], &pGVCpu->vmm.s.u.aLoggers[iLogger]);
+ }
+
+ /*
+ * Free logger buffer memory.
+ */
+ RTR0MemObjFree(pGVM->vmmr0.s.hMapObjReleaseLogger, false /*fFreeMappings*/);
+ pGVM->vmmr0.s.hMapObjReleaseLogger = NIL_RTR0MEMOBJ;
+ RTR0MemObjFree(pGVM->vmmr0.s.hMemObjReleaseLogger, true /*fFreeMappings*/);
+ pGVM->vmmr0.s.hMemObjReleaseLogger = NIL_RTR0MEMOBJ;
+
+ RTR0MemObjFree(pGVM->vmmr0.s.hMapObjLogger, false /*fFreeMappings*/);
+ pGVM->vmmr0.s.hMapObjLogger = NIL_RTR0MEMOBJ;
+ RTR0MemObjFree(pGVM->vmmr0.s.hMemObjLogger, true /*fFreeMappings*/);
+ pGVM->vmmr0.s.hMemObjLogger = NIL_RTR0MEMOBJ;
+
+ /*
+ * Free log flusher related stuff.
+ */
+ RTSpinlockDestroy(pGVM->vmmr0.s.LogFlusher.hSpinlock);
+ pGVM->vmmr0.s.LogFlusher.hSpinlock = NIL_RTSPINLOCK;
+ RTSemEventDestroy(pGVM->vmmr0.s.LogFlusher.hEvent);
+ pGVM->vmmr0.s.LogFlusher.hEvent = NIL_RTSEMEVENT;
+}
+
+
+/*********************************************************************************************************************************
+* Assertions *
+*********************************************************************************************************************************/
+
+/**
+ * Installs a notification callback for ring-0 assertions.
+ *
+ * @param pVCpu The cross context virtual CPU structure.
+ * @param pfnCallback Pointer to the callback.
+ * @param pvUser The user argument.
+ *
+ * @return VBox status code.
+ */
+VMMR0_INT_DECL(int) VMMR0AssertionSetNotification(PVMCPUCC pVCpu, PFNVMMR0ASSERTIONNOTIFICATION pfnCallback, RTR0PTR pvUser)
+{
+ AssertPtrReturn(pVCpu, VERR_INVALID_POINTER);
+ AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+
+ if (!pVCpu->vmmr0.s.pfnAssertCallback)
+ {
+ pVCpu->vmmr0.s.pfnAssertCallback = pfnCallback;
+ pVCpu->vmmr0.s.pvAssertCallbackUser = pvUser;
+ return VINF_SUCCESS;
+ }
+ return VERR_ALREADY_EXISTS;
+}
+
+
+/**
+ * Removes the ring-0 callback.
+ *
+ * @param pVCpu The cross context virtual CPU structure.
+ */
+VMMR0_INT_DECL(void) VMMR0AssertionRemoveNotification(PVMCPUCC pVCpu)
+{
+ pVCpu->vmmr0.s.pfnAssertCallback = NULL;
+ pVCpu->vmmr0.s.pvAssertCallbackUser = NULL;
+}
+
+
+/**
+ * Checks whether there is a ring-0 callback notification active.
+ *
+ * @param pVCpu The cross context virtual CPU structure.
+ * @returns true if there the notification is active, false otherwise.
+ */
+VMMR0_INT_DECL(bool) VMMR0AssertionIsNotificationSet(PVMCPUCC pVCpu)
+{
+ return pVCpu->vmmr0.s.pfnAssertCallback != NULL;
+}
+
+
+/*
+ * Jump back to ring-3 if we're the EMT and the longjmp is armed.
+ *
+ * @returns true if the breakpoint should be hit, false if it should be ignored.
+ */
+DECLEXPORT(bool) RTCALL RTAssertShouldPanic(void)
+{
+#if 0
+ return true;
+#else
+ PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
+ if (pVM)
+ {
+ PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+ if (pVCpu)
+ {
+# ifdef RT_ARCH_X86
+ if (pVCpu->vmmr0.s.AssertJmpBuf.eip)
+# else
+ if (pVCpu->vmmr0.s.AssertJmpBuf.rip)
+# endif
+ {
+ if (pVCpu->vmmr0.s.pfnAssertCallback)
+ pVCpu->vmmr0.s.pfnAssertCallback(pVCpu, pVCpu->vmmr0.s.pvAssertCallbackUser);
+ int rc = vmmR0CallRing3LongJmp(&pVCpu->vmmr0.s.AssertJmpBuf, VERR_VMM_RING0_ASSERTION);
+ return RT_FAILURE_NP(rc);
+ }
+ }
+ }
+# ifdef RT_OS_LINUX
+ return true;
+# else
+ return false;
+# endif
+#endif
+}
+
+
+/*
+ * Override this so we can push it up to ring-3.
+ */
+DECLEXPORT(void) RTCALL RTAssertMsg1Weak(const char *pszExpr, unsigned uLine, const char *pszFile, const char *pszFunction)
+{
+ /*
+ * To host kernel log/whatever.
+ */
+ SUPR0Printf("!!R0-Assertion Failed!!\n"
+ "Expression: %s\n"
+ "Location : %s(%d) %s\n",
+ pszExpr, pszFile, uLine, pszFunction);
+
+ /*
+ * To the log.
+ */
+ LogAlways(("\n!!R0-Assertion Failed!!\n"
+ "Expression: %s\n"
+ "Location : %s(%d) %s\n",
+ pszExpr, pszFile, uLine, pszFunction));
+
+ /*
+ * To the global VMM buffer.
+ */
+ PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
+ if (pVM)
+ RTStrPrintf(pVM->vmm.s.szRing0AssertMsg1, sizeof(pVM->vmm.s.szRing0AssertMsg1),
+ "\n!!R0-Assertion Failed!!\n"
+ "Expression: %.*s\n"
+ "Location : %s(%d) %s\n",
+ sizeof(pVM->vmm.s.szRing0AssertMsg1) / 4 * 3, pszExpr,
+ pszFile, uLine, pszFunction);
+
+ /*
+ * Continue the normal way.
+ */
+ RTAssertMsg1(pszExpr, uLine, pszFile, pszFunction);
+}
+
+
+/**
+ * Callback for RTLogFormatV which writes to the ring-3 log port.
+ * See PFNLOGOUTPUT() for details.
+ */
+static DECLCALLBACK(size_t) rtLogOutput(void *pv, const char *pachChars, size_t cbChars)
+{
+ for (size_t i = 0; i < cbChars; i++)
+ {
+ LogAlways(("%c", pachChars[i])); NOREF(pachChars);
+ }
+
+ NOREF(pv);
+ return cbChars;
+}
+
+
+/*
+ * Override this so we can push it up to ring-3.
+ */
+DECLEXPORT(void) RTCALL RTAssertMsg2WeakV(const char *pszFormat, va_list va)
+{
+ va_list vaCopy;
+
+ /*
+ * Push the message to the loggers.
+ */
+ PRTLOGGER pLog = RTLogRelGetDefaultInstance();
+ if (pLog)
+ {
+ va_copy(vaCopy, va);
+ RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
+ va_end(vaCopy);
+ }
+ pLog = RTLogGetDefaultInstance(); /* Don't initialize it here... */
+ if (pLog)
+ {
+ va_copy(vaCopy, va);
+ RTLogFormatV(rtLogOutput, pLog, pszFormat, vaCopy);
+ va_end(vaCopy);
+ }
+
+ /*
+ * Push it to the global VMM buffer.
+ */
+ PVMCC pVM = GVMMR0GetVMByEMT(NIL_RTNATIVETHREAD);
+ if (pVM)
+ {
+ va_copy(vaCopy, va);
+ RTStrPrintfV(pVM->vmm.s.szRing0AssertMsg2, sizeof(pVM->vmm.s.szRing0AssertMsg2), pszFormat, vaCopy);
+ va_end(vaCopy);
+ }
+
+ /*
+ * Continue the normal way.
+ */
+ RTAssertMsg2V(pszFormat, va);
+}
+
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-14 03:47:59 +0000