Adding upstream version 7.0.14-dfsg.upstream/7.0.14-dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 867 insertions, 0 deletions

diff --git a/src/VBox/VMM/VMMR0/PDMR0Device.cpp b/src/VBox/VMM/VMMR0/PDMR0Device.cpp
new file mode 100644
index 00000000..946a4bb3
--- /dev/null
+++ b/src/VBox/VMM/VMMR0/PDMR0Device.cpp
@@ -0,0 +1,867 @@
+/* $Id: PDMR0Device.cpp $ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, R0 Device parts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#define PDMPCIDEV_INCLUDE_PRIVATE  /* Hack to get pdmpcidevint.h included at the right point. */
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/gvmm.h>
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/msi.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/mem.h>
+#include <iprt/memobj.h>
+#include <iprt/process.h>
+#include <iprt/string.h>
+
+#include "dtrace/VBoxVMM.h"
+#include "PDMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+extern DECLEXPORT(const PDMDEVHLPR0)    g_pdmR0DevHlp;
+#ifdef VBOX_WITH_DBGF_TRACING
+extern DECLEXPORT(const PDMDEVHLPR0)    g_pdmR0DevHlpTracing;
+#endif
+extern DECLEXPORT(const PDMPICHLP)      g_pdmR0PicHlp;
+extern DECLEXPORT(const PDMIOAPICHLP)   g_pdmR0IoApicHlp;
+extern DECLEXPORT(const PDMPCIHLPR0)    g_pdmR0PciHlp;
+extern DECLEXPORT(const PDMIOMMUHLPR0)  g_pdmR0IommuHlp;
+extern DECLEXPORT(const PDMHPETHLPR0)   g_pdmR0HpetHlp;
+extern DECLEXPORT(const PDMPCIRAWHLPR0) g_pdmR0PciRawHlp;
+RT_C_DECLS_END
+
+/** List of PDMDEVMODREGR0 structures protected by the loader lock. */
+static RTLISTANCHOR g_PDMDevModList;
+
+
+/**
+ * Pointer to the ring-0 device registrations for VMMR0.
+ */
+static const PDMDEVREGR0 *g_apVMM0DevRegs[] =
+{
+    &g_DeviceAPIC,
+};
+
+/**
+ * Module device registration record for VMMR0.
+ */
+static PDMDEVMODREGR0 g_VBoxDDR0ModDevReg =
+{
+    /* .u32Version = */ PDM_DEVMODREGR0_VERSION,
+    /* .cDevRegs = */   RT_ELEMENTS(g_apVMM0DevRegs),
+    /* .papDevRegs = */ &g_apVMM0DevRegs[0],
+    /* .hMod = */       NULL,
+    /* .ListEntry = */  { NULL, NULL },
+};
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+
+
+/**
+ * Initializes the global ring-0 PDM data.
+ */
+VMMR0_INT_DECL(void) PDMR0Init(void *hMod)
+{
+    RTListInit(&g_PDMDevModList);
+    g_VBoxDDR0ModDevReg.hMod = hMod;
+    RTListAppend(&g_PDMDevModList, &g_VBoxDDR0ModDevReg.ListEntry);
+}
+
+
+/**
+ * Used by PDMR0CleanupVM to destroy a device instance.
+ *
+ * This is done during VM cleanup so that we're sure there are no active threads
+ * inside the device code.
+ *
+ * @param   pGVM        The global (ring-0) VM structure.
+ * @param   pDevIns     The device instance.
+ * @param   idxR0Device The device instance handle.
+ */
+static int pdmR0DeviceDestroy(PGVM pGVM, PPDMDEVINSR0 pDevIns, uint32_t idxR0Device)
+{
+    /*
+     * Assert sanity.
+     */
+    Assert(idxR0Device < pGVM->pdmr0.s.cDevInstances);
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    Assert(pDevIns->u32Version == PDM_DEVINSR0_VERSION);
+    Assert(pDevIns->Internal.s.idxR0Device == idxR0Device);
+
+    /*
+     * Call the final destructor if there is one.
+     */
+    if (pDevIns->pReg->pfnFinalDestruct)
+        pDevIns->pReg->pfnFinalDestruct(pDevIns);
+    pDevIns->u32Version = ~PDM_DEVINSR0_VERSION;
+
+    /*
+     * Remove the device from the instance table.
+     */
+    Assert(pGVM->pdmr0.s.apDevInstances[idxR0Device] == pDevIns);
+    pGVM->pdmr0.s.apDevInstances[idxR0Device] = NULL;
+    if (idxR0Device + 1 == pGVM->pdmr0.s.cDevInstances)
+        pGVM->pdmr0.s.cDevInstances = idxR0Device;
+
+    /*
+     * Free the DBGF tracing tracking structures if necessary.
+     */
+    if (pDevIns->Internal.s.hDbgfTraceEvtSrc != NIL_DBGFTRACEREVTSRC)
+    {
+        RTR0MemObjFree(pDevIns->Internal.s.hDbgfTraceObj, true);
+        pDevIns->Internal.s.hDbgfTraceObj = NIL_RTR0MEMOBJ;
+    }
+
+    /*
+     * Free the ring-3 mapping and instance memory.
+     */
+    RTR0MEMOBJ hMemObj = pDevIns->Internal.s.hMapObj;
+    pDevIns->Internal.s.hMapObj = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(hMemObj, true);
+
+    hMemObj = pDevIns->Internal.s.hMemObj;
+    pDevIns->Internal.s.hMemObj = NIL_RTR0MEMOBJ;
+    RTR0MemObjFree(hMemObj, true);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initializes the per-VM data for the PDM.
+ *
+ * This is called from under the GVMM lock, so it only need to initialize the
+ * data so PDMR0CleanupVM and others will work smoothly.
+ *
+ * @param   pGVM    Pointer to the global VM structure.
+ */
+VMMR0_INT_DECL(void) PDMR0InitPerVMData(PGVM pGVM)
+{
+    AssertCompile(sizeof(pGVM->pdm.s) <= sizeof(pGVM->pdm.padding));
+    AssertCompile(sizeof(pGVM->pdmr0.s) <= sizeof(pGVM->pdmr0.padding));
+
+    pGVM->pdmr0.s.cDevInstances = 0;
+}
+
+
+/**
+ * Cleans up any loose ends before the GVM structure is destroyed.
+ */
+VMMR0_INT_DECL(void) PDMR0CleanupVM(PGVM pGVM)
+{
+    uint32_t i = pGVM->pdmr0.s.cDevInstances;
+    while (i-- > 0)
+    {
+        PPDMDEVINSR0 pDevIns = pGVM->pdmr0.s.apDevInstances[i];
+        if (pDevIns)
+            pdmR0DeviceDestroy(pGVM, pDevIns, i);
+    }
+
+    i = pGVM->pdmr0.s.cQueues;
+    while (i-- > 0)
+    {
+        if (pGVM->pdmr0.s.aQueues[i].pQueue != NULL)
+            pdmR0QueueDestroy(pGVM, i);
+    }
+}
+
+
+/**
+ * Worker for PDMR0DeviceCreate that does the actual instantiation.
+ *
+ * Allocates a memory object and divides it up as follows:
+ * @verbatim
+   --------------------------------------
+   ring-0 devins
+   --------------------------------------
+   ring-0 instance data
+   --------------------------------------
+   ring-0 PCI device data (optional) ??
+   --------------------------------------
+   page alignment padding
+   --------------------------------------
+   ring-3 devins
+   --------------------------------------
+   ring-3 instance data
+   --------------------------------------
+   ring-3 PCI device data (optional) ??
+   --------------------------------------
+  [page alignment padding                ] -
+  [--------------------------------------]  \
+  [raw-mode devins                       ]   \
+  [--------------------------------------]   - Optional, only when raw-mode is enabled.
+  [raw-mode instance data                ]   /
+  [--------------------------------------]  /
+  [raw-mode PCI device data (optional)?? ] -
+   --------------------------------------
+   shared instance data
+   --------------------------------------
+   default crit section
+   --------------------------------------
+   shared PCI device data (optional)
+   --------------------------------------
+   @endverbatim
+ *
+ * @returns VBox status code.
+ * @param   pGVM             The global (ring-0) VM structure.
+ * @param   pDevReg          The device registration structure.
+ * @param   iInstance        The device instance number.
+ * @param   cbInstanceR3     The size of the ring-3 instance data.
+ * @param   cbInstanceRC     The size of the raw-mode instance data.
+ * @param   hMod             The module implementing the device.
+ * @param   hDbgfTraceEvtSrc The DBGF tarcer event source handle.
+ * @param   RCPtrMapping     The raw-mode context mapping address, NIL_RTGCPTR if
+ *                           not to include raw-mode.
+ * @param   ppDevInsR3       Where to return the ring-3 device instance address.
+ * @thread  EMT(0)
+ */
+static int pdmR0DeviceCreateWorker(PGVM pGVM, PCPDMDEVREGR0 pDevReg, uint32_t iInstance, uint32_t cbInstanceR3,
+                                   uint32_t cbInstanceRC, RTRGPTR RCPtrMapping, DBGFTRACEREVTSRC hDbgfTraceEvtSrc,
+                                   void *hMod, PPDMDEVINSR3 *ppDevInsR3)
+{
+    /*
+     * Check that the instance number isn't a duplicate.
+     */
+    for (size_t i = 0; i < pGVM->pdmr0.s.cDevInstances; i++)
+    {
+        PPDMDEVINS pCur = pGVM->pdmr0.s.apDevInstances[i];
+        AssertLogRelReturn(!pCur || pCur->pReg != pDevReg || pCur->iInstance != iInstance, VERR_DUPLICATE);
+    }
+
+    /*
+     * Figure out how much memory we need and allocate it.
+     */
+    uint32_t const cbRing0     = RT_ALIGN_32(RT_UOFFSETOF(PDMDEVINSR0, achInstanceData) + pDevReg->cbInstanceCC, HOST_PAGE_SIZE);
+    uint32_t const cbRing3     = RT_ALIGN_32(RT_UOFFSETOF(PDMDEVINSR3, achInstanceData) + cbInstanceR3,
+                                             RCPtrMapping != NIL_RTRGPTR ? HOST_PAGE_SIZE : 64);
+    uint32_t const cbRC        = RCPtrMapping != NIL_RTRGPTR ? 0
+                               : RT_ALIGN_32(RT_UOFFSETOF(PDMDEVINSRC, achInstanceData) + cbInstanceRC, 64);
+    uint32_t const cbShared    = RT_ALIGN_32(pDevReg->cbInstanceShared, 64);
+    uint32_t const cbCritSect  = RT_ALIGN_32(sizeof(PDMCRITSECT), 64);
+    uint32_t const cbMsixState = RT_ALIGN_32(pDevReg->cMaxMsixVectors * 16 + (pDevReg->cMaxMsixVectors + 7) / 8, _4K);
+    uint32_t const cbPciDev    = RT_ALIGN_32(RT_UOFFSETOF_DYN(PDMPCIDEV, abMsixState[cbMsixState]), 64);
+    uint32_t const cPciDevs    = RT_MIN(pDevReg->cMaxPciDevices, 8);
+    uint32_t const cbPciDevs   = cbPciDev * cPciDevs;
+    uint32_t const cbTotal     = RT_ALIGN_32(cbRing0 + cbRing3 + cbRC + cbShared + cbCritSect + cbPciDevs, HOST_PAGE_SIZE);
+    AssertLogRelMsgReturn(cbTotal <= PDM_MAX_DEVICE_INSTANCE_SIZE,
+                          ("Instance of '%s' is too big: cbTotal=%u, max %u\n",
+                           pDevReg->szName, cbTotal, PDM_MAX_DEVICE_INSTANCE_SIZE),
+                          VERR_OUT_OF_RANGE);
+
+    RTR0MEMOBJ hMemObj;
+    int rc = RTR0MemObjAllocPage(&hMemObj, cbTotal, false /*fExecutable*/);
+    if (RT_FAILURE(rc))
+        return rc;
+    RT_BZERO(RTR0MemObjAddress(hMemObj), cbTotal);
+
+    /* Map it. */
+    RTR0MEMOBJ hMapObj;
+    rc = RTR0MemObjMapUserEx(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ | RTMEM_PROT_WRITE, RTR0ProcHandleSelf(),
+                             cbRing0, cbTotal - cbRing0);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMDEVINSR0        pDevIns   = (PPDMDEVINSR0)RTR0MemObjAddress(hMemObj);
+        struct PDMDEVINSR3 *pDevInsR3 = (struct PDMDEVINSR3 *)((uint8_t *)pDevIns + cbRing0);
+
+        /*
+         * Initialize the ring-0 instance.
+         */
+        pDevIns->u32Version             = PDM_DEVINSR0_VERSION;
+        pDevIns->iInstance              = iInstance;
+#ifdef VBOX_WITH_DBGF_TRACING
+        pDevIns->pHlpR0                 = hDbgfTraceEvtSrc == NIL_DBGFTRACEREVTSRC ? &g_pdmR0DevHlp : &g_pdmR0DevHlpTracing;
+#else
+        pDevIns->pHlpR0                 = &g_pdmR0DevHlp;
+#endif
+        pDevIns->pvInstanceDataR0       = (uint8_t *)pDevIns + cbRing0 + cbRing3 + cbRC;
+        pDevIns->pvInstanceDataForR0    = &pDevIns->achInstanceData[0];
+        pDevIns->pCritSectRoR0          = (PPDMCRITSECT)((uint8_t *)pDevIns->pvInstanceDataR0 + cbShared);
+        pDevIns->pReg                   = pDevReg;
+        pDevIns->pDevInsForR3           = RTR0MemObjAddressR3(hMapObj);
+        pDevIns->pDevInsForR3R0         = pDevInsR3;
+        pDevIns->pvInstanceDataForR3R0  = &pDevInsR3->achInstanceData[0];
+        pDevIns->cbPciDev               = cbPciDev;
+        pDevIns->cPciDevs               = cPciDevs;
+        for (uint32_t iPciDev = 0; iPciDev < cPciDevs; iPciDev++)
+        {
+            /* Note! PDMDevice.cpp has a copy of this code.  Keep in sync. */
+            PPDMPCIDEV pPciDev = (PPDMPCIDEV)((uint8_t *)pDevIns->pCritSectRoR0 + cbCritSect + cbPciDev * iPciDev);
+            if (iPciDev < RT_ELEMENTS(pDevIns->apPciDevs))
+                pDevIns->apPciDevs[iPciDev] = pPciDev;
+            pPciDev->cbConfig           = _4K;
+            pPciDev->cbMsixState        = cbMsixState;
+            pPciDev->idxSubDev          = (uint16_t)iPciDev;
+            pPciDev->Int.s.idxSubDev    = (uint16_t)iPciDev;
+            pPciDev->u32Magic           = PDMPCIDEV_MAGIC;
+        }
+        pDevIns->Internal.s.pGVM        = pGVM;
+        pDevIns->Internal.s.pRegR0      = pDevReg;
+        pDevIns->Internal.s.hMod        = hMod;
+        pDevIns->Internal.s.hMemObj     = hMemObj;
+        pDevIns->Internal.s.hMapObj     = hMapObj;
+        pDevIns->Internal.s.pInsR3R0    = pDevInsR3;
+        pDevIns->Internal.s.pIntR3R0    = &pDevInsR3->Internal.s;
+        pDevIns->Internal.s.hDbgfTraceEvtSrc = hDbgfTraceEvtSrc;
+
+        /*
+         * Initialize the ring-3 instance data as much as we can.
+         * Note! PDMDevice.cpp does this job for ring-3 only devices.  Keep in sync.
+         */
+        pDevInsR3->u32Version           = PDM_DEVINSR3_VERSION;
+        pDevInsR3->iInstance            = iInstance;
+        pDevInsR3->cbRing3              = cbTotal - cbRing0;
+        pDevInsR3->fR0Enabled           = true;
+        pDevInsR3->fRCEnabled           = RCPtrMapping != NIL_RTRGPTR;
+        pDevInsR3->pvInstanceDataR3     = pDevIns->pDevInsForR3 + cbRing3 + cbRC;
+        pDevInsR3->pvInstanceDataForR3  = pDevIns->pDevInsForR3 + RT_UOFFSETOF(PDMDEVINSR3, achInstanceData);
+        pDevInsR3->pCritSectRoR3        = pDevIns->pDevInsForR3 + cbRing3 + cbRC + cbShared;
+        pDevInsR3->pDevInsR0RemoveMe    = pDevIns;
+        pDevInsR3->pvInstanceDataR0     = pDevIns->pvInstanceDataR0;
+        pDevInsR3->pvInstanceDataRC     = RCPtrMapping == NIL_RTRGPTR
+                                        ? NIL_RTRGPTR : pDevIns->pDevInsForRC + RT_UOFFSETOF(PDMDEVINSRC, achInstanceData);
+        pDevInsR3->pDevInsForRC         = pDevIns->pDevInsForRC;
+        pDevInsR3->pDevInsForRCR3       = pDevIns->pDevInsForR3 + cbRing3;
+        pDevInsR3->pDevInsForRCR3       = pDevInsR3->pDevInsForRCR3 + RT_UOFFSETOF(PDMDEVINSRC, achInstanceData);
+        pDevInsR3->cbPciDev             = cbPciDev;
+        pDevInsR3->cPciDevs             = cPciDevs;
+        for (uint32_t i = 0; i < RT_MIN(cPciDevs, RT_ELEMENTS(pDevIns->apPciDevs)); i++)
+            pDevInsR3->apPciDevs[i] = pDevInsR3->pCritSectRoR3 + cbCritSect + cbPciDev * i;
+
+        pDevInsR3->Internal.s.pVMR3     = pGVM->pVMR3;
+        pDevInsR3->Internal.s.fIntFlags = RCPtrMapping == NIL_RTRGPTR ? PDMDEVINSINT_FLAGS_R0_ENABLED
+                                        : PDMDEVINSINT_FLAGS_R0_ENABLED | PDMDEVINSINT_FLAGS_RC_ENABLED;
+        pDevInsR3->Internal.s.hDbgfTraceEvtSrc = hDbgfTraceEvtSrc;
+
+        /*
+         * Initialize the raw-mode instance data as much as possible.
+         */
+        if (RCPtrMapping != NIL_RTRGPTR)
+        {
+            struct PDMDEVINSRC *pDevInsRC = RCPtrMapping == NIL_RTRGPTR ? NULL
+                                          : (struct PDMDEVINSRC *)((uint8_t *)pDevIns + cbRing0 + cbRing3);
+
+            pDevIns->pDevInsForRC           = RCPtrMapping;
+            pDevIns->pDevInsForRCR0         = pDevInsRC;
+            pDevIns->pvInstanceDataForRCR0  = &pDevInsRC->achInstanceData[0];
+
+            pDevInsRC->u32Version           = PDM_DEVINSRC_VERSION;
+            pDevInsRC->iInstance            = iInstance;
+            pDevInsRC->pvInstanceDataRC     = pDevIns->pDevInsForRC + cbRC;
+            pDevInsRC->pvInstanceDataForRC  = pDevIns->pDevInsForRC + RT_UOFFSETOF(PDMDEVINSRC, achInstanceData);
+            pDevInsRC->pCritSectRoRC        = pDevIns->pDevInsForRC + cbRC + cbShared;
+            pDevInsRC->cbPciDev             = cbPciDev;
+            pDevInsRC->cPciDevs             = cPciDevs;
+            for (uint32_t i = 0; i < RT_MIN(cPciDevs, RT_ELEMENTS(pDevIns->apPciDevs)); i++)
+                pDevInsRC->apPciDevs[i] = pDevInsRC->pCritSectRoRC + cbCritSect + cbPciDev * i;
+
+            pDevInsRC->Internal.s.pVMRC     = pGVM->pVMRC;
+        }
+
+        /*
+         * If the device is being traced we have to set up a single page for tracking
+         * I/O and MMIO region registrations so we can inject our own handlers.
+         */
+        if (hDbgfTraceEvtSrc != NIL_DBGFTRACEREVTSRC)
+        {
+            pDevIns->Internal.s.hDbgfTraceObj = NIL_RTR0MEMOBJ;
+            rc = RTR0MemObjAllocPage(&pDevIns->Internal.s.hDbgfTraceObj, PDM_MAX_DEVICE_DBGF_TRACING_TRACK, false /*fExecutable*/);
+            if (RT_SUCCESS(rc))
+            {
+                pDevIns->Internal.s.paDbgfTraceTrack      = (PPDMDEVINSDBGFTRACK)RTR0MemObjAddress(pDevIns->Internal.s.hDbgfTraceObj);
+                pDevIns->Internal.s.idxDbgfTraceTrackNext = 0;
+                pDevIns->Internal.s.cDbgfTraceTrackMax    = PDM_MAX_DEVICE_DBGF_TRACING_TRACK / sizeof(PDMDEVINSDBGFTRACK);
+                RT_BZERO(pDevIns->Internal.s.paDbgfTraceTrack, PDM_MAX_DEVICE_DBGF_TRACING_TRACK);
+            }
+        }
+
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Add to the device instance array and set its handle value.
+             */
+            AssertCompile(sizeof(pGVM->pdmr0.padding) == sizeof(pGVM->pdmr0));
+            uint32_t idxR0Device = pGVM->pdmr0.s.cDevInstances;
+            if (idxR0Device < RT_ELEMENTS(pGVM->pdmr0.s.apDevInstances))
+            {
+                pGVM->pdmr0.s.apDevInstances[idxR0Device]    = pDevIns;
+                pGVM->pdmr0.s.cDevInstances                  = idxR0Device + 1;
+                pGVM->pdm.s.apDevRing0Instances[idxR0Device] = pDevIns->pDevInsForR3;
+                pDevIns->Internal.s.idxR0Device   = idxR0Device;
+                pDevInsR3->Internal.s.idxR0Device = idxR0Device;
+
+                /*
+                 * Call the early constructor if present.
+                 */
+                if (pDevReg->pfnEarlyConstruct)
+                    rc = pDevReg->pfnEarlyConstruct(pDevIns);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * We're done.
+                     */
+                    *ppDevInsR3 = RTR0MemObjAddressR3(hMapObj);
+                    return rc;
+                }
+
+                /*
+                 * Bail out.
+                 */
+                if (pDevIns->pReg->pfnFinalDestruct)
+                    pDevIns->pReg->pfnFinalDestruct(pDevIns);
+
+                pGVM->pdmr0.s.apDevInstances[idxR0Device] = NULL;
+                Assert(pGVM->pdmr0.s.cDevInstances == idxR0Device + 1);
+                pGVM->pdmr0.s.cDevInstances = idxR0Device;
+            }
+        }
+
+        if (   hDbgfTraceEvtSrc != NIL_DBGFTRACEREVTSRC
+            && pDevIns->Internal.s.hDbgfTraceObj != NIL_RTR0MEMOBJ)
+            RTR0MemObjFree(pDevIns->Internal.s.hDbgfTraceObj, true);
+
+        RTR0MemObjFree(hMapObj, true);
+    }
+    RTR0MemObjFree(hMemObj, true);
+    return rc;
+}
+
+
+/**
+ * Used by ring-3 PDM to create a device instance that operates both in ring-3
+ * and ring-0.
+ *
+ * Creates an instance of a device (for both ring-3 and ring-0, and optionally
+ * raw-mode context).
+ *
+ * @returns VBox status code.
+ * @param   pGVM    The global (ring-0) VM structure.
+ * @param   pReq    Pointer to the request buffer.
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) PDMR0DeviceCreateReqHandler(PGVM pGVM, PPDMDEVICECREATEREQ pReq)
+{
+    LogFlow(("PDMR0DeviceCreateReqHandler: %s in %s\n", pReq->szDevName, pReq->szModName));
+
+    /*
+     * Validate the request.
+     */
+    AssertReturn(pReq->Hdr.cbReq == sizeof(*pReq), VERR_INVALID_PARAMETER);
+    pReq->pDevInsR3 = NIL_RTR3PTR;
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+
+    AssertReturn(pReq->fFlags           != 0, VERR_INVALID_FLAGS);
+    AssertReturn(pReq->fClass           != 0, VERR_WRONG_TYPE);
+    AssertReturn(pReq->uSharedVersion   != 0, VERR_INVALID_PARAMETER);
+    AssertReturn(pReq->cbInstanceShared != 0, VERR_INVALID_PARAMETER);
+    size_t const cchDevName = RTStrNLen(pReq->szDevName, sizeof(pReq->szDevName));
+    AssertReturn(cchDevName < sizeof(pReq->szDevName), VERR_NO_STRING_TERMINATOR);
+    AssertReturn(cchDevName > 0, VERR_EMPTY_STRING);
+    AssertReturn(cchDevName < RT_SIZEOFMEMB(PDMDEVREG, szName), VERR_NOT_FOUND);
+
+    size_t const cchModName = RTStrNLen(pReq->szModName, sizeof(pReq->szModName));
+    AssertReturn(cchModName < sizeof(pReq->szModName), VERR_NO_STRING_TERMINATOR);
+    AssertReturn(cchModName > 0, VERR_EMPTY_STRING);
+    AssertReturn(pReq->cbInstanceShared <= PDM_MAX_DEVICE_INSTANCE_SIZE, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cbInstanceR3 <= PDM_MAX_DEVICE_INSTANCE_SIZE, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cbInstanceRC <= PDM_MAX_DEVICE_INSTANCE_SIZE, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->iInstance < 1024, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->iInstance < pReq->cMaxInstances, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cMaxPciDevices <= 8, VERR_OUT_OF_RANGE);
+    AssertReturn(pReq->cMaxMsixVectors <= VBOX_MSIX_MAX_ENTRIES, VERR_OUT_OF_RANGE);
+
+    /*
+     * Reference the module.
+     */
+    void *hMod = NULL;
+    rc = SUPR0LdrModByName(pGVM->pSession, pReq->szModName, &hMod);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("PDMR0DeviceCreateReqHandler: SUPR0LdrModByName(,%s,) failed: %Rrc\n", pReq->szModName, rc));
+        return rc;
+    }
+
+    /*
+     * Look for the the module and the device registration structure.
+     */
+    int rcLock = SUPR0LdrLock(pGVM->pSession);
+    AssertRC(rc);
+
+    rc = VERR_NOT_FOUND;
+    PPDMDEVMODREGR0 pMod;
+    RTListForEach(&g_PDMDevModList, pMod, PDMDEVMODREGR0, ListEntry)
+    {
+        if (pMod->hMod == hMod)
+        {
+            /*
+             * Found the module. We can drop the loader lock now before we
+             * search the devices it registers.
+             */
+            if (RT_SUCCESS(rcLock))
+            {
+                rcLock = SUPR0LdrUnlock(pGVM->pSession);
+                AssertRC(rcLock);
+            }
+            rcLock = VERR_ALREADY_RESET;
+
+            PCPDMDEVREGR0 *papDevRegs = pMod->papDevRegs;
+            size_t         i          = pMod->cDevRegs;
+            while (i-- > 0)
+            {
+                PCPDMDEVREGR0 pDevReg = papDevRegs[i];
+                LogFlow(("PDMR0DeviceCreateReqHandler: candidate #%u: %s %#x\n", i, pReq->szDevName, pDevReg->u32Version));
+                if (   PDM_VERSION_ARE_COMPATIBLE(pDevReg->u32Version, PDM_DEVREGR0_VERSION)
+                    && pDevReg->szName[cchDevName] == '\0'
+                    && memcmp(pDevReg->szName, pReq->szDevName, cchDevName) == 0)
+                {
+
+                    /*
+                     * Found the device, now check whether it matches the ring-3 registration.
+                     */
+                    if (   pReq->uSharedVersion   == pDevReg->uSharedVersion
+                        && pReq->cbInstanceShared == pDevReg->cbInstanceShared
+                        && pReq->cbInstanceRC     == pDevReg->cbInstanceRC
+                        && pReq->fFlags           == pDevReg->fFlags
+                        && pReq->fClass           == pDevReg->fClass
+                        && pReq->cMaxInstances    == pDevReg->cMaxInstances
+                        && pReq->cMaxPciDevices   == pDevReg->cMaxPciDevices
+                        && pReq->cMaxMsixVectors  == pDevReg->cMaxMsixVectors)
+                    {
+                        rc = pdmR0DeviceCreateWorker(pGVM, pDevReg, pReq->iInstance, pReq->cbInstanceR3, pReq->cbInstanceRC,
+                                                     NIL_RTRCPTR /** @todo new raw-mode */, pReq->hDbgfTracerEvtSrc,
+                                                     hMod, &pReq->pDevInsR3);
+                        if (RT_SUCCESS(rc))
+                            hMod = NULL; /* keep the module reference */
+                    }
+                    else
+                    {
+                        LogRel(("PDMR0DeviceCreate: Ring-3 does not match ring-0 device registration (%s):\n"
+                                "    uSharedVersion: %#x vs %#x\n"
+                                "  cbInstanceShared: %#x vs %#x\n"
+                                "      cbInstanceRC: %#x vs %#x\n"
+                                "            fFlags: %#x vs %#x\n"
+                                "            fClass: %#x vs %#x\n"
+                                "     cMaxInstances: %#x vs %#x\n"
+                                "    cMaxPciDevices: %#x vs %#x\n"
+                                "   cMaxMsixVectors: %#x vs %#x\n"
+                                ,
+                                pReq->szDevName,
+                                pReq->uSharedVersion,   pDevReg->uSharedVersion,
+                                pReq->cbInstanceShared, pDevReg->cbInstanceShared,
+                                pReq->cbInstanceRC,     pDevReg->cbInstanceRC,
+                                pReq->fFlags,           pDevReg->fFlags,
+                                pReq->fClass,           pDevReg->fClass,
+                                pReq->cMaxInstances,    pDevReg->cMaxInstances,
+                                pReq->cMaxPciDevices,   pDevReg->cMaxPciDevices,
+                                pReq->cMaxMsixVectors,  pDevReg->cMaxMsixVectors));
+                        rc = VERR_INCOMPATIBLE_CONFIG;
+                    }
+                }
+            }
+            break;
+        }
+    }
+
+    if (RT_SUCCESS_NP(rcLock))
+    {
+        rcLock = SUPR0LdrUnlock(pGVM->pSession);
+        AssertRC(rcLock);
+    }
+    SUPR0LdrModRelease(pGVM->pSession, hMod);
+    return rc;
+}
+
+
+/**
+ * Used by ring-3 PDM to call standard ring-0 device methods.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    The global (ring-0) VM structure.
+ * @param   pReq    Pointer to the request buffer.
+ * @param   idCpu   The ID of the calling EMT.
+ * @thread  EMT(0), except for PDMDEVICEGENCALL_REQUEST which can be any EMT.
+ */
+VMMR0_INT_DECL(int) PDMR0DeviceGenCallReqHandler(PGVM pGVM, PPDMDEVICEGENCALLREQ pReq, VMCPUID idCpu)
+{
+    /*
+     * Validate the request.
+     */
+    AssertReturn(pReq->Hdr.cbReq == sizeof(*pReq), VERR_INVALID_PARAMETER);
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, idCpu);
+    AssertRCReturn(rc, rc);
+
+    AssertReturn(pReq->idxR0Device < pGVM->pdmr0.s.cDevInstances, VERR_INVALID_HANDLE);
+    PPDMDEVINSR0 pDevIns = pGVM->pdmr0.s.apDevInstances[pReq->idxR0Device];
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(pDevIns->pDevInsForR3 == pReq->pDevInsR3, VERR_INVALID_HANDLE);
+
+    /*
+     * Make the call.
+     */
+    rc = VINF_SUCCESS /*VINF_NOT_IMPLEMENTED*/;
+    switch (pReq->enmCall)
+    {
+        case PDMDEVICEGENCALL_CONSTRUCT:
+            AssertMsgBreakStmt(pGVM->enmVMState < VMSTATE_CREATED, ("enmVMState=%d\n", pGVM->enmVMState), rc = VERR_INVALID_STATE);
+            AssertReturn(idCpu == 0,  VERR_VM_THREAD_NOT_EMT);
+            if (pDevIns->pReg->pfnConstruct)
+                rc = pDevIns->pReg->pfnConstruct(pDevIns);
+            break;
+
+        case PDMDEVICEGENCALL_DESTRUCT:
+            AssertMsgBreakStmt(pGVM->enmVMState < VMSTATE_CREATED || pGVM->enmVMState >= VMSTATE_DESTROYING,
+                               ("enmVMState=%d\n", pGVM->enmVMState), rc = VERR_INVALID_STATE);
+            AssertReturn(idCpu == 0,  VERR_VM_THREAD_NOT_EMT);
+            if (pDevIns->pReg->pfnDestruct)
+            {
+                pDevIns->pReg->pfnDestruct(pDevIns);
+                rc = VINF_SUCCESS;
+            }
+            break;
+
+        case PDMDEVICEGENCALL_REQUEST:
+            if (pDevIns->pReg->pfnRequest)
+                rc = pDevIns->pReg->pfnRequest(pDevIns, pReq->Params.Req.uReq, pReq->Params.Req.uArg);
+            else
+                rc = VERR_INVALID_FUNCTION;
+            break;
+
+        default:
+            AssertMsgFailed(("enmCall=%d\n", pReq->enmCall));
+            rc = VERR_INVALID_FUNCTION;
+            break;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Legacy device mode compatiblity.
+ *
+ * @returns VBox status code.
+ * @param   pGVM    The global (ring-0) VM structure.
+ * @param   pReq    Pointer to the request buffer.
+ * @thread  EMT(0)
+ */
+VMMR0_INT_DECL(int) PDMR0DeviceCompatSetCritSectReqHandler(PGVM pGVM, PPDMDEVICECOMPATSETCRITSECTREQ pReq)
+{
+    /*
+     * Validate the request.
+     */
+    AssertReturn(pReq->Hdr.cbReq == sizeof(*pReq), VERR_INVALID_PARAMETER);
+
+    int rc = GVMMR0ValidateGVMandEMT(pGVM, 0);
+    AssertRCReturn(rc, rc);
+
+    AssertReturn(pReq->idxR0Device < pGVM->pdmr0.s.cDevInstances, VERR_INVALID_HANDLE);
+    PPDMDEVINSR0 pDevIns = pGVM->pdmr0.s.apDevInstances[pReq->idxR0Device];
+    AssertPtrReturn(pDevIns, VERR_INVALID_HANDLE);
+    AssertReturn(pDevIns->pDevInsForR3 == pReq->pDevInsR3, VERR_INVALID_HANDLE);
+
+    AssertReturn(pGVM->enmVMState == VMSTATE_CREATING, VERR_INVALID_STATE);
+
+    /*
+     * The critical section address can be in a few different places:
+     *      1. shared data.
+     *      2. nop section.
+     *      3. pdm critsect.
+     */
+    PPDMCRITSECT pCritSect;
+    if (pReq->pCritSectR3 == pGVM->pVMR3 + RT_UOFFSETOF(VM, pdm.s.NopCritSect))
+    {
+        pCritSect = &pGVM->pdm.s.NopCritSect;
+        Log(("PDMR0DeviceCompatSetCritSectReqHandler: Nop - %p %#x\n", pCritSect, pCritSect->s.Core.u32Magic));
+    }
+    else if (pReq->pCritSectR3 == pGVM->pVMR3 + RT_UOFFSETOF(VM, pdm.s.CritSect))
+    {
+        pCritSect = &pGVM->pdm.s.CritSect;
+        Log(("PDMR0DeviceCompatSetCritSectReqHandler: PDM - %p %#x\n", pCritSect, pCritSect->s.Core.u32Magic));
+    }
+    else
+    {
+        size_t offCritSect = pReq->pCritSectR3 - pDevIns->pDevInsForR3R0->pvInstanceDataR3;
+        AssertLogRelMsgReturn(   offCritSect                       <  pDevIns->pReg->cbInstanceShared
+                              && offCritSect + sizeof(PDMCRITSECT) <= pDevIns->pReg->cbInstanceShared,
+                              ("offCritSect=%p pCritSectR3=%p cbInstanceShared=%#x (%s)\n",
+                               offCritSect, pReq->pCritSectR3, pDevIns->pReg->cbInstanceShared, pDevIns->pReg->szName),
+                              VERR_INVALID_POINTER);
+        pCritSect = (PPDMCRITSECT)((uint8_t *)pDevIns->pvInstanceDataR0 + offCritSect);
+        Log(("PDMR0DeviceCompatSetCritSectReqHandler: custom - %#x/%p %#x\n", offCritSect, pCritSect, pCritSect->s.Core.u32Magic));
+    }
+    AssertLogRelMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
+                          ("cs=%p magic=%#x dev=%s\n", pCritSect, pCritSect->s.Core.u32Magic, pDevIns->pReg->szName),
+                          VERR_INVALID_MAGIC);
+
+    /*
+     * Make the update.
+     */
+    pDevIns->pCritSectRoR0 = pCritSect;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers the device implementations living in a module.
+ *
+ * This should normally only be called during ModuleInit().  The should be a
+ * call to PDMR0DeviceDeregisterModule from the ModuleTerm() function to undo
+ * the effects of this call.
+ *
+ * @returns VBox status code.
+ * @param   hMod            The module handle of the module being registered.
+ * @param   pModReg         The module registration structure.  This will be
+ *                          used directly so it must live as long as the module
+ *                          and be writable.
+ *
+ * @note    Caller must own the loader lock!
+ */
+VMMR0DECL(int) PDMR0DeviceRegisterModule(void *hMod, PPDMDEVMODREGR0 pModReg)
+{
+    /*
+     * Validate the input.
+     */
+    AssertPtrReturn(hMod, VERR_INVALID_HANDLE);
+    Assert(SUPR0LdrIsLockOwnerByMod(hMod, true));
+
+    AssertPtrReturn(pModReg, VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(PDM_VERSION_ARE_COMPATIBLE(pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          ("pModReg->u32Version=%#x vs %#x\n", pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          VERR_VERSION_MISMATCH);
+    AssertLogRelMsgReturn(pModReg->cDevRegs <= 256 && pModReg->cDevRegs > 0, ("cDevRegs=%u\n", pModReg->cDevRegs),
+                          VERR_OUT_OF_RANGE);
+    AssertLogRelMsgReturn(pModReg->hMod == NULL, ("hMod=%p\n", pModReg->hMod), VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(pModReg->ListEntry.pNext == NULL, ("pNext=%p\n", pModReg->ListEntry.pNext), VERR_INVALID_PARAMETER);
+    AssertLogRelMsgReturn(pModReg->ListEntry.pPrev == NULL, ("pPrev=%p\n", pModReg->ListEntry.pPrev), VERR_INVALID_PARAMETER);
+
+    for (size_t i = 0; i < pModReg->cDevRegs; i++)
+    {
+        PCPDMDEVREGR0 pDevReg = pModReg->papDevRegs[i];
+        AssertLogRelMsgReturn(RT_VALID_PTR(pDevReg), ("[%u]: %p\n", i, pDevReg), VERR_INVALID_POINTER);
+        AssertLogRelMsgReturn(PDM_VERSION_ARE_COMPATIBLE(pDevReg->u32Version, PDM_DEVREGR0_VERSION),
+                              ("pDevReg->u32Version=%#x vs %#x\n", pModReg->u32Version, PDM_DEVREGR0_VERSION), VERR_VERSION_MISMATCH);
+        AssertLogRelMsgReturn(RT_VALID_PTR(pDevReg->pszDescription), ("[%u]: %p\n", i, pDevReg->pszDescription), VERR_INVALID_POINTER);
+        AssertLogRelMsgReturn(pDevReg->uReserved0     == 0, ("[%u]: %#x\n", i, pDevReg->uReserved0),     VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->fClass         != 0, ("[%u]: %#x\n", i, pDevReg->fClass),         VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->fFlags         != 0, ("[%u]: %#x\n", i, pDevReg->fFlags),         VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->cMaxInstances   > 0, ("[%u]: %#x\n", i, pDevReg->cMaxInstances),  VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->cMaxPciDevices <= 8, ("[%u]: %#x\n", i, pDevReg->cMaxPciDevices), VERR_INVALID_PARAMETER);
+        AssertLogRelMsgReturn(pDevReg->cMaxMsixVectors <= VBOX_MSIX_MAX_ENTRIES,
+                              ("[%u]: %#x\n", i, pDevReg->cMaxMsixVectors), VERR_INVALID_PARAMETER);
+
+        /* The name must be printable ascii and correctly terminated. */
+        for (size_t off = 0; off < RT_ELEMENTS(pDevReg->szName); off++)
+        {
+            char ch = pDevReg->szName[off];
+            AssertLogRelMsgReturn(RT_C_IS_PRINT(ch) || (ch == '\0' && off > 0),
+                                  ("[%u]: off=%u  szName: %.*Rhxs\n", i, off, sizeof(pDevReg->szName), &pDevReg->szName[0]),
+                                  VERR_INVALID_NAME);
+            if (ch == '\0')
+                break;
+        }
+    }
+
+    /*
+     * Add it, assuming we're being called at ModuleInit/ModuleTerm time only, or
+     * that the caller has already taken the loader lock.
+     */
+    pModReg->hMod = hMod;
+    RTListAppend(&g_PDMDevModList, &pModReg->ListEntry);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters the device implementations living in a module.
+ *
+ * This should normally only be called during ModuleTerm().
+ *
+ * @returns VBox status code.
+ * @param   hMod            The module handle of the module being registered.
+ * @param   pModReg         The module registration structure.  This will be
+ *                          used directly so it must live as long as the module
+ *                          and be writable.
+ *
+ * @note    Caller must own the loader lock!
+ */
+VMMR0DECL(int) PDMR0DeviceDeregisterModule(void *hMod, PPDMDEVMODREGR0 pModReg)
+{
+    /*
+     * Validate the input.
+     */
+    AssertPtrReturn(hMod, VERR_INVALID_HANDLE);
+    Assert(SUPR0LdrIsLockOwnerByMod(hMod, true));
+
+    AssertPtrReturn(pModReg, VERR_INVALID_POINTER);
+    AssertLogRelMsgReturn(PDM_VERSION_ARE_COMPATIBLE(pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          ("pModReg->u32Version=%#x vs %#x\n", pModReg->u32Version, PDM_DEVMODREGR0_VERSION),
+                          VERR_VERSION_MISMATCH);
+    AssertLogRelMsgReturn(pModReg->hMod == hMod || pModReg->hMod == NULL, ("pModReg->hMod=%p vs %p\n", pModReg->hMod, hMod),
+                          VERR_INVALID_PARAMETER);
+
+    /*
+     * Unlink the registration record and return it to virgin conditions.  Ignore
+     * the call if not registered.
+     */
+    if (pModReg->hMod)
+    {
+        pModReg->hMod = NULL;
+        RTListNodeRemove(&pModReg->ListEntry);
+        pModReg->ListEntry.pNext = NULL;
+        pModReg->ListEntry.pPrev = NULL;
+        return VINF_SUCCESS;
+    }
+    return VWRN_NOT_FOUND;
+}
+