summaryrefslogtreecommitdiffstats
path: root/src/VBox/Runtime/r0drv/linux
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:17:27 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:17:27 +0000
commitf215e02bf85f68d3a6106c2a1f4f7f063f819064 (patch)
tree6bb5b92c046312c4e95ac2620b10ddf482d3fa8b /src/VBox/Runtime/r0drv/linux
parentInitial commit. (diff)
downloadvirtualbox-f215e02bf85f68d3a6106c2a1f4f7f063f819064.tar.xz
virtualbox-f215e02bf85f68d3a6106c2a1f4f7f063f819064.zip
Adding upstream version 7.0.14-dfsg.upstream/7.0.14-dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/VBox/Runtime/r0drv/linux')
-rw-r--r--src/VBox/Runtime/r0drv/linux/Makefile.kup0
-rw-r--r--src/VBox/Runtime/r0drv/linux/RTLogWriteDebugger-r0drv-linux.c53
-rw-r--r--src/VBox/Runtime/r0drv/linux/alloc-r0drv-linux.c257
-rw-r--r--src/VBox/Runtime/r0drv/linux/assert-r0drv-linux.c84
-rw-r--r--src/VBox/Runtime/r0drv/linux/initterm-r0drv-linux.c138
-rw-r--r--src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c2104
-rw-r--r--src/VBox/Runtime/r0drv/linux/memuserkernel-r0drv-linux.c191
-rw-r--r--src/VBox/Runtime/r0drv/linux/mp-r0drv-linux.c640
-rw-r--r--src/VBox/Runtime/r0drv/linux/mpnotification-r0drv-linux.c258
-rw-r--r--src/VBox/Runtime/r0drv/linux/process-r0drv-linux.c59
-rw-r--r--src/VBox/Runtime/r0drv/linux/rtStrFormatKernelAddress-r0drv-linux.c66
-rw-r--r--src/VBox/Runtime/r0drv/linux/semevent-r0drv-linux.c296
-rw-r--r--src/VBox/Runtime/r0drv/linux/semeventmulti-r0drv-linux.c361
-rw-r--r--src/VBox/Runtime/r0drv/linux/semfastmutex-r0drv-linux.c167
-rw-r--r--src/VBox/Runtime/r0drv/linux/semmutex-r0drv-linux.c431
-rw-r--r--src/VBox/Runtime/r0drv/linux/spinlock-r0drv-linux.c196
-rw-r--r--src/VBox/Runtime/r0drv/linux/string.h70
-rw-r--r--src/VBox/Runtime/r0drv/linux/the-linux-kernel.h494
-rw-r--r--src/VBox/Runtime/r0drv/linux/thread-r0drv-linux.c285
-rw-r--r--src/VBox/Runtime/r0drv/linux/thread2-r0drv-linux.c243
-rw-r--r--src/VBox/Runtime/r0drv/linux/threadctxhooks-r0drv-linux.c341
-rw-r--r--src/VBox/Runtime/r0drv/linux/time-r0drv-linux.c221
-rw-r--r--src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c1739
-rw-r--r--src/VBox/Runtime/r0drv/linux/waitqueue-r0drv-linux.h302
24 files changed, 8996 insertions, 0 deletions
diff --git a/src/VBox/Runtime/r0drv/linux/Makefile.kup b/src/VBox/Runtime/r0drv/linux/Makefile.kup
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/Makefile.kup
diff --git a/src/VBox/Runtime/r0drv/linux/RTLogWriteDebugger-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/RTLogWriteDebugger-r0drv-linux.c
new file mode 100644
index 00000000..7de10556
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/RTLogWriteDebugger-r0drv-linux.c
@@ -0,0 +1,53 @@
+/* $Id: RTLogWriteDebugger-r0drv-linux.c $ */
+/** @file
+ * IPRT - Log To Debugger, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/log.h>
+
+
+RTDECL(void) RTLogWriteDebugger(const char *pch, size_t cb)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ printk("%.*s", (int)cb, pch);
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+RT_EXPORT_SYMBOL(RTLogWriteDebugger);
+
diff --git a/src/VBox/Runtime/r0drv/linux/alloc-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/alloc-r0drv-linux.c
new file mode 100644
index 00000000..47410cca
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/alloc-r0drv-linux.c
@@ -0,0 +1,257 @@
+/* $Id: alloc-r0drv-linux.c $ */
+/** @file
+ * IPRT - Memory Allocation, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/mem.h>
+
+#include <iprt/assert.h>
+#include <iprt/errcore.h>
+#include "r0drv/alloc-r0drv.h"
+
+#include "internal/initterm.h"
+
+
+
+/**
+ * OS specific allocation function.
+ */
+DECLHIDDEN(int) rtR0MemAllocEx(size_t cb, uint32_t fFlags, PRTMEMHDR *ppHdr)
+{
+ PRTMEMHDR pHdr;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Allocate.
+ */
+ if (
+#if 1 /* vmalloc has serious performance issues, avoid it. */
+ cb <= PAGE_SIZE*16 - sizeof(*pHdr)
+#else
+ cb <= PAGE_SIZE
+#endif
+ || (fFlags & RTMEMHDR_FLAG_ANY_CTX)
+ )
+ {
+ fFlags |= RTMEMHDR_FLAG_KMALLOC;
+ pHdr = kmalloc(cb + sizeof(*pHdr),
+ fFlags & RTMEMHDR_FLAG_ANY_CTX_ALLOC ? GFP_ATOMIC | __GFP_NOWARN : GFP_KERNEL | __GFP_NOWARN);
+ if (RT_UNLIKELY( !pHdr
+ && cb > PAGE_SIZE
+ && !(fFlags & RTMEMHDR_FLAG_ANY_CTX) ))
+ {
+ fFlags &= ~RTMEMHDR_FLAG_KMALLOC;
+ pHdr = vmalloc(cb + sizeof(*pHdr));
+ }
+ }
+ else
+ pHdr = vmalloc(cb + sizeof(*pHdr));
+ if (RT_LIKELY(pHdr))
+ {
+ /*
+ * Initialize.
+ */
+ pHdr->u32Magic = RTMEMHDR_MAGIC;
+ pHdr->fFlags = fFlags;
+ pHdr->cb = cb;
+ pHdr->cbReq = cb;
+
+ *ppHdr = pHdr;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+}
+
+
+/**
+ * OS specific free function.
+ */
+DECLHIDDEN(void) rtR0MemFree(PRTMEMHDR pHdr)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ pHdr->u32Magic += 1;
+ if (pHdr->fFlags & RTMEMHDR_FLAG_KMALLOC)
+ kfree(pHdr);
+ else
+ vfree(pHdr);
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
+
+
+/**
+ * Compute order. Some functions allocate 2^order pages.
+ *
+ * @returns order.
+ * @param cPages Number of pages.
+ */
+static int CalcPowerOf2Order(unsigned long cPages)
+{
+ int iOrder;
+ unsigned long cTmp;
+
+ for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
+ ;
+ if (cPages & ~(1 << iOrder))
+ ++iOrder;
+
+ return iOrder;
+}
+
+
+/**
+ * Allocates physical contiguous memory (below 4GB).
+ * The allocation is page aligned and the content is undefined.
+ *
+ * @returns Pointer to the memory block. This is page aligned.
+ * @param pPhys Where to store the physical address.
+ * @param cb The allocation size in bytes. This is always
+ * rounded up to PAGE_SIZE.
+ */
+RTR0DECL(void *) RTMemContAlloc(PRTCCPHYS pPhys, size_t cb)
+{
+ int cOrder;
+ unsigned cPages;
+ struct page *paPages;
+ void *pvRet;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * validate input.
+ */
+ AssertPtr(pPhys);
+ Assert(cb > 0);
+
+ /*
+ * Allocate page pointer array.
+ */
+ cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+ cPages = cb >> PAGE_SHIFT;
+ cOrder = CalcPowerOf2Order(cPages);
+#if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
+ /* ZONE_DMA32: 0-4GB */
+ paPages = alloc_pages(GFP_DMA32 | __GFP_NOWARN, cOrder);
+ if (!paPages)
+#endif
+#ifdef RT_ARCH_AMD64
+ /* ZONE_DMA; 0-16MB */
+ paPages = alloc_pages(GFP_DMA | __GFP_NOWARN, cOrder);
+#else
+ /* ZONE_NORMAL: 0-896MB */
+ paPages = alloc_pages(GFP_USER | __GFP_NOWARN, cOrder);
+#endif
+ if (paPages)
+ {
+ /*
+ * Reserve the pages and mark them executable.
+ */
+ unsigned iPage;
+ for (iPage = 0; iPage < cPages; iPage++)
+ {
+ Assert(!PageHighMem(&paPages[iPage]));
+ if (iPage + 1 < cPages)
+ {
+ AssertMsg( (uintptr_t)phys_to_virt(page_to_phys(&paPages[iPage])) + PAGE_SIZE
+ == (uintptr_t)phys_to_virt(page_to_phys(&paPages[iPage + 1]))
+ && page_to_phys(&paPages[iPage]) + PAGE_SIZE
+ == page_to_phys(&paPages[iPage + 1]),
+ ("iPage=%i cPages=%u [0]=%#llx,%p [1]=%#llx,%p\n", iPage, cPages,
+ (long long)page_to_phys(&paPages[iPage]), phys_to_virt(page_to_phys(&paPages[iPage])),
+ (long long)page_to_phys(&paPages[iPage + 1]), phys_to_virt(page_to_phys(&paPages[iPage + 1])) ));
+ }
+
+ SetPageReserved(&paPages[iPage]);
+ }
+ *pPhys = page_to_phys(paPages);
+ pvRet = phys_to_virt(page_to_phys(paPages));
+ }
+ else
+ pvRet = NULL;
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return pvRet;
+}
+RT_EXPORT_SYMBOL(RTMemContAlloc);
+
+
+/**
+ * Frees memory allocated using RTMemContAlloc().
+ *
+ * @param pv Pointer to return from RTMemContAlloc().
+ * @param cb The cb parameter passed to RTMemContAlloc().
+ */
+RTR0DECL(void) RTMemContFree(void *pv, size_t cb)
+{
+ if (pv)
+ {
+ int cOrder;
+ unsigned cPages;
+ unsigned iPage;
+ struct page *paPages;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /* validate */
+ AssertMsg(!((uintptr_t)pv & PAGE_OFFSET_MASK), ("pv=%p\n", pv));
+ Assert(cb > 0);
+
+ /* calc order and get pages */
+ cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+ cPages = cb >> PAGE_SHIFT;
+ cOrder = CalcPowerOf2Order(cPages);
+ paPages = virt_to_page(pv);
+
+ /*
+ * Restore page attributes freeing the pages.
+ */
+ for (iPage = 0; iPage < cPages; iPage++)
+ {
+ ClearPageReserved(&paPages[iPage]);
+ }
+ __free_pages(paPages, cOrder);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ }
+}
+RT_EXPORT_SYMBOL(RTMemContFree);
+
diff --git a/src/VBox/Runtime/r0drv/linux/assert-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/assert-r0drv-linux.c
new file mode 100644
index 00000000..a7968d06
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/assert-r0drv-linux.c
@@ -0,0 +1,84 @@
+/* $Id: assert-r0drv-linux.c $ */
+/** @file
+ * IPRT - Assertion Workers, Ring-0 Drivers, Linux.
+ */
+
+/*
+ * Copyright (C) 2007-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/assert.h>
+#include <iprt/log.h>
+#include <iprt/string.h>
+#include <iprt/stdarg.h>
+#include <iprt/asm.h>
+
+#include "internal/assert.h"
+
+
+DECLHIDDEN(void) rtR0AssertNativeMsg1(const char *pszExpr, unsigned uLine, const char *pszFile, const char *pszFunction)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ printk(KERN_EMERG
+ "\r\n!!Assertion Failed!!\r\n"
+ "Expression: %s\r\n"
+ "Location : %s(%d) %s\r\n",
+ pszExpr, pszFile, uLine, pszFunction);
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
+
+DECLHIDDEN(void) rtR0AssertNativeMsg2V(bool fInitial, const char *pszFormat, va_list va)
+{
+ char szMsg[256];
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ RTStrPrintfV(szMsg, sizeof(szMsg) - 1, pszFormat, va);
+ szMsg[sizeof(szMsg) - 1] = '\0';
+ printk(KERN_EMERG "%s", szMsg);
+
+ NOREF(fInitial);
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
+
+RTR0DECL(void) RTR0AssertPanicSystem(void)
+{
+ panic("%s%s", g_szRTAssertMsg1, g_szRTAssertMsg2);
+}
+RT_EXPORT_SYMBOL(RTR0AssertPanicSystem);
+
diff --git a/src/VBox/Runtime/r0drv/linux/initterm-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/initterm-r0drv-linux.c
new file mode 100644
index 00000000..cbdb324a
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/initterm-r0drv-linux.c
@@ -0,0 +1,138 @@
+/* $Id: initterm-r0drv-linux.c $ */
+/** @file
+ * IPRT - Initialization & Termination, R0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/errcore.h>
+#include <iprt/assert.h>
+#include "internal/initterm.h"
+
+
+/*********************************************************************************************************************************
+* Global Variables *
+*********************************************************************************************************************************/
+/** The IPRT work queue. */
+#if RTLNX_VER_MIN(2,5,41)
+static struct workqueue_struct *g_prtR0LnxWorkQueue;
+#else
+static DECLARE_TASK_QUEUE(g_rtR0LnxWorkQueue);
+#endif
+
+
+/**
+ * Pushes an item onto the IPRT work queue.
+ *
+ * @param pWork The work item.
+ * @param pfnWorker The callback function. It will be called back
+ * with @a pWork as argument.
+ */
+DECLHIDDEN(void) rtR0LnxWorkqueuePush(RTR0LNXWORKQUEUEITEM *pWork, void (*pfnWorker)(RTR0LNXWORKQUEUEITEM *))
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+#if RTLNX_VER_MIN(2,5,41)
+# if RTLNX_VER_MIN(2,6,20)
+ INIT_WORK(pWork, pfnWorker);
+# else
+ INIT_WORK(pWork, (void (*)(void *))pfnWorker, pWork);
+# endif
+ queue_work(g_prtR0LnxWorkQueue, pWork);
+#else
+ INIT_TQUEUE(pWork, (void (*)(void *))pfnWorker, pWork);
+ queue_task(pWork, &g_rtR0LnxWorkQueue);
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
+
+/**
+ * Flushes all items in the IPRT work queue.
+ *
+ * @remarks This is mostly for 2.4.x compatability. Must not be called from
+ * atomic contexts or with unncessary locks held.
+ */
+DECLHIDDEN(void) rtR0LnxWorkqueueFlush(void)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+#if RTLNX_VER_MIN(2,5,41)
+ flush_workqueue(g_prtR0LnxWorkQueue);
+#else
+ run_task_queue(&g_rtR0LnxWorkQueue);
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
+
+DECLHIDDEN(int) rtR0InitNative(void)
+{
+ int rc = VINF_SUCCESS;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+#if RTLNX_VER_MIN(2,5,41)
+ #if RTLNX_VER_MIN(2,6,13)
+ g_prtR0LnxWorkQueue = create_workqueue("iprt-VBoxWQueue");
+ #else
+ g_prtR0LnxWorkQueue = create_workqueue("iprt-VBoxQ");
+ #endif
+ if (!g_prtR0LnxWorkQueue)
+ rc = VERR_NO_MEMORY;
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+DECLHIDDEN(void) rtR0TermNative(void)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ rtR0LnxWorkqueueFlush();
+#if RTLNX_VER_MIN(2,5,41)
+ destroy_workqueue(g_prtR0LnxWorkQueue);
+ g_prtR0LnxWorkQueue = NULL;
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
diff --git a/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c
new file mode 100644
index 00000000..8342fbf8
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/memobj-r0drv-linux.c
@@ -0,0 +1,2104 @@
+/* $Id: memobj-r0drv-linux.c $ */
+/** @file
+ * IPRT - Ring-0 Memory Objects, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+
+#include <iprt/memobj.h>
+#include <iprt/assert.h>
+#include <iprt/err.h>
+#include <iprt/log.h>
+#include <iprt/mem.h>
+#include <iprt/process.h>
+#include <iprt/string.h>
+#include "internal/memobj.h"
+#include "internal/iprt.h"
+
+
+/*********************************************************************************************************************************
+* Defined Constants And Macros *
+*********************************************************************************************************************************/
+/* early 2.6 kernels */
+#ifndef PAGE_SHARED_EXEC
+# define PAGE_SHARED_EXEC PAGE_SHARED
+#endif
+#ifndef PAGE_READONLY_EXEC
+# define PAGE_READONLY_EXEC PAGE_READONLY
+#endif
+
+/** @def IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
+ * Whether we use alloc_vm_area (3.2+) for executable memory.
+ * This is a must for 5.8+, but we enable it all the way back to 3.2.x for
+ * better W^R compliance (fExecutable flag). */
+#if RTLNX_VER_RANGE(3,2,0, 5,10,0) || defined(DOXYGEN_RUNNING)
+# define IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
+#endif
+/** @def IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
+ * alloc_vm_area was removed with 5.10 so we have to resort to a different way
+ * to allocate executable memory.
+ * It would be possible to remove IPRT_USE_ALLOC_VM_AREA_FOR_EXEC and use
+ * this path execlusively for 3.2+ but no time to test it really works on every
+ * supported kernel, so better play safe for now.
+ */
+#if RTLNX_VER_MIN(5,10,0) || defined(DOXYGEN_RUNNING)
+# define IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
+#endif
+
+/*
+ * 2.6.29+ kernels don't work with remap_pfn_range() anymore because
+ * track_pfn_vma_new() is apparently not defined for non-RAM pages.
+ * It should be safe to use vm_insert_page() older kernels as well.
+ */
+#if RTLNX_VER_MIN(2,6,23)
+# define VBOX_USE_INSERT_PAGE
+#endif
+#if defined(CONFIG_X86_PAE) \
+ && ( defined(HAVE_26_STYLE_REMAP_PAGE_RANGE) \
+ || RTLNX_VER_RANGE(2,6,0, 2,6,11) )
+# define VBOX_USE_PAE_HACK
+#endif
+
+/* gfp_t was introduced in 2.6.14, define it for earlier. */
+#if RTLNX_VER_MAX(2,6,14)
+# define gfp_t unsigned
+#endif
+
+/*
+ * Wrappers around mmap_lock/mmap_sem difference.
+ */
+#if RTLNX_VER_MIN(5,8,0)
+# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_lock)
+# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_lock)
+# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_lock)
+# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_lock)
+#else
+# define LNX_MM_DOWN_READ(a_pMm) down_read(&(a_pMm)->mmap_sem)
+# define LNX_MM_UP_READ(a_pMm) up_read(&(a_pMm)->mmap_sem)
+# define LNX_MM_DOWN_WRITE(a_pMm) down_write(&(a_pMm)->mmap_sem)
+# define LNX_MM_UP_WRITE(a_pMm) up_write(&(a_pMm)->mmap_sem)
+#endif
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+/**
+ * The Linux version of the memory object structure.
+ */
+typedef struct RTR0MEMOBJLNX
+{
+ /** The core structure. */
+ RTR0MEMOBJINTERNAL Core;
+ /** Set if the allocation is contiguous.
+ * This means it has to be given back as one chunk. */
+ bool fContiguous;
+ /** Set if executable allocation. */
+ bool fExecutable;
+ /** Set if we've vmap'ed the memory into ring-0. */
+ bool fMappedToRing0;
+ /** This is non-zero if large page allocation. */
+ uint8_t cLargePageOrder;
+#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
+ /** Return from alloc_vm_area() that we now need to use for executable
+ * memory. */
+ struct vm_struct *pArea;
+ /** PTE array that goes along with pArea (must be freed). */
+ pte_t **papPtesForArea;
+#endif
+ /** The pages in the apPages array. */
+ size_t cPages;
+ /** Array of struct page pointers. (variable size) */
+ struct page *apPages[1];
+} RTR0MEMOBJLNX;
+/** Pointer to the linux memory object. */
+typedef RTR0MEMOBJLNX *PRTR0MEMOBJLNX;
+
+
+static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx);
+
+
+/**
+ * Helper that converts from a RTR0PROCESS handle to a linux task.
+ *
+ * @returns The corresponding Linux task.
+ * @param R0Process IPRT ring-0 process handle.
+ */
+static struct task_struct *rtR0ProcessToLinuxTask(RTR0PROCESS R0Process)
+{
+ /** @todo fix rtR0ProcessToLinuxTask!! */
+ /** @todo many (all?) callers currently assume that we return 'current'! */
+ return R0Process == RTR0ProcHandleSelf() ? current : NULL;
+}
+
+
+/**
+ * Compute order. Some functions allocate 2^order pages.
+ *
+ * @returns order.
+ * @param cPages Number of pages.
+ */
+static int rtR0MemObjLinuxOrder(size_t cPages)
+{
+ int iOrder;
+ size_t cTmp;
+
+ for (iOrder = 0, cTmp = cPages; cTmp >>= 1; ++iOrder)
+ ;
+ if (cPages & ~((size_t)1 << iOrder))
+ ++iOrder;
+
+ return iOrder;
+}
+
+
+/**
+ * Converts from RTMEM_PROT_* to Linux PAGE_*.
+ *
+ * @returns Linux page protection constant.
+ * @param fProt The IPRT protection mask.
+ * @param fKernel Whether it applies to kernel or user space.
+ */
+static pgprot_t rtR0MemObjLinuxConvertProt(unsigned fProt, bool fKernel)
+{
+ switch (fProt)
+ {
+ default:
+ AssertMsgFailed(("%#x %d\n", fProt, fKernel)); RT_FALL_THRU();
+ case RTMEM_PROT_NONE:
+ return PAGE_NONE;
+
+ case RTMEM_PROT_READ:
+ return fKernel ? PAGE_KERNEL_RO : PAGE_READONLY;
+
+ case RTMEM_PROT_WRITE:
+ case RTMEM_PROT_WRITE | RTMEM_PROT_READ:
+ return fKernel ? PAGE_KERNEL : PAGE_SHARED;
+
+ case RTMEM_PROT_EXEC:
+ case RTMEM_PROT_EXEC | RTMEM_PROT_READ:
+#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
+ if (fKernel)
+ {
+ pgprot_t fPg = MY_PAGE_KERNEL_EXEC;
+ pgprot_val(fPg) &= ~_PAGE_RW;
+ return fPg;
+ }
+ return PAGE_READONLY_EXEC;
+#else
+ return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_READONLY_EXEC;
+#endif
+
+ case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC:
+ case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC | RTMEM_PROT_READ:
+ return fKernel ? MY_PAGE_KERNEL_EXEC : PAGE_SHARED_EXEC;
+ }
+}
+
+
+/**
+ * Worker for rtR0MemObjNativeReserveUser and rtR0MemObjNativerMapUser that creates
+ * an empty user space mapping.
+ *
+ * We acquire the mmap_sem/mmap_lock of the task!
+ *
+ * @returns Pointer to the mapping.
+ * (void *)-1 on failure.
+ * @param R3PtrFixed (RTR3PTR)-1 if anywhere, otherwise a specific location.
+ * @param cb The size of the mapping.
+ * @param uAlignment The alignment of the mapping.
+ * @param pTask The Linux task to create this mapping in.
+ * @param fProt The RTMEM_PROT_* mask.
+ */
+static void *rtR0MemObjLinuxDoMmap(RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, struct task_struct *pTask, unsigned fProt)
+{
+ unsigned fLnxProt;
+ unsigned long ulAddr;
+
+ Assert(pTask == current); /* do_mmap */
+ RT_NOREF_PV(pTask);
+
+ /*
+ * Convert from IPRT protection to mman.h PROT_ and call do_mmap.
+ */
+ fProt &= (RTMEM_PROT_NONE | RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC);
+ if (fProt == RTMEM_PROT_NONE)
+ fLnxProt = PROT_NONE;
+ else
+ {
+ fLnxProt = 0;
+ if (fProt & RTMEM_PROT_READ)
+ fLnxProt |= PROT_READ;
+ if (fProt & RTMEM_PROT_WRITE)
+ fLnxProt |= PROT_WRITE;
+ if (fProt & RTMEM_PROT_EXEC)
+ fLnxProt |= PROT_EXEC;
+ }
+
+ if (R3PtrFixed != (RTR3PTR)-1)
+ {
+#if RTLNX_VER_MIN(3,5,0)
+ ulAddr = vm_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, 0);
+#else
+ LNX_MM_DOWN_WRITE(pTask->mm);
+ ulAddr = do_mmap(NULL, R3PtrFixed, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, 0);
+ LNX_MM_UP_WRITE(pTask->mm);
+#endif
+ }
+ else
+ {
+#if RTLNX_VER_MIN(3,5,0)
+ ulAddr = vm_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS, 0);
+#else
+ LNX_MM_DOWN_WRITE(pTask->mm);
+ ulAddr = do_mmap(NULL, 0, cb, fLnxProt, MAP_SHARED | MAP_ANONYMOUS, 0);
+ LNX_MM_UP_WRITE(pTask->mm);
+#endif
+ if ( !(ulAddr & ~PAGE_MASK)
+ && (ulAddr & (uAlignment - 1)))
+ {
+ /** @todo implement uAlignment properly... We'll probably need to make some dummy mappings to fill
+ * up alignment gaps. This is of course complicated by fragmentation (which we might have cause
+ * ourselves) and further by there begin two mmap strategies (top / bottom). */
+ /* For now, just ignore uAlignment requirements... */
+ }
+ }
+
+
+ if (ulAddr & ~PAGE_MASK) /* ~PAGE_MASK == PAGE_OFFSET_MASK */
+ return (void *)-1;
+ return (void *)ulAddr;
+}
+
+
+/**
+ * Worker that destroys a user space mapping.
+ * Undoes what rtR0MemObjLinuxDoMmap did.
+ *
+ * We acquire the mmap_sem/mmap_lock of the task!
+ *
+ * @param pv The ring-3 mapping.
+ * @param cb The size of the mapping.
+ * @param pTask The Linux task to destroy this mapping in.
+ */
+static void rtR0MemObjLinuxDoMunmap(void *pv, size_t cb, struct task_struct *pTask)
+{
+#if RTLNX_VER_MIN(3,5,0)
+ Assert(pTask == current); RT_NOREF_PV(pTask);
+ vm_munmap((unsigned long)pv, cb);
+#elif defined(USE_RHEL4_MUNMAP)
+ LNX_MM_DOWN_WRITE(pTask->mm);
+ do_munmap(pTask->mm, (unsigned long)pv, cb, 0); /* should it be 1 or 0? */
+ LNX_MM_UP_WRITE(pTask->mm);
+#else
+ LNX_MM_DOWN_WRITE(pTask->mm);
+ do_munmap(pTask->mm, (unsigned long)pv, cb);
+ LNX_MM_UP_WRITE(pTask->mm);
+#endif
+}
+
+
+/**
+ * Internal worker that allocates physical pages and creates the memory object for them.
+ *
+ * @returns IPRT status code.
+ * @param ppMemLnx Where to store the memory object pointer.
+ * @param enmType The object type.
+ * @param cb The number of bytes to allocate.
+ * @param uAlignment The alignment of the physical memory.
+ * Only valid if fContiguous == true, ignored otherwise.
+ * @param fFlagsLnx The page allocation flags (GPFs).
+ * @param fContiguous Whether the allocation must be contiguous.
+ * @param fExecutable Whether the memory must be executable.
+ * @param rcNoMem What to return when we're out of pages.
+ * @param pszTag Allocation tag used for statistics and such.
+ */
+static int rtR0MemObjLinuxAllocPages(PRTR0MEMOBJLNX *ppMemLnx, RTR0MEMOBJTYPE enmType, size_t cb,
+ size_t uAlignment, gfp_t fFlagsLnx, bool fContiguous, bool fExecutable, int rcNoMem,
+ const char *pszTag)
+{
+ size_t iPage;
+ size_t const cPages = cb >> PAGE_SHIFT;
+ struct page *paPages;
+
+ /*
+ * Allocate a memory object structure that's large enough to contain
+ * the page pointer array.
+ */
+ PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), enmType,
+ NULL, cb, pszTag);
+ if (!pMemLnx)
+ return VERR_NO_MEMORY;
+ pMemLnx->Core.fFlags |= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC;
+ pMemLnx->cPages = cPages;
+
+ if (cPages > 255)
+ {
+# ifdef __GFP_REPEAT
+ /* Try hard to allocate the memory, but the allocation attempt might fail. */
+ fFlagsLnx |= __GFP_REPEAT;
+# endif
+# ifdef __GFP_NOMEMALLOC
+ /* Introduced with Linux 2.6.12: Don't use emergency reserves */
+ fFlagsLnx |= __GFP_NOMEMALLOC;
+# endif
+ }
+
+ /*
+ * Allocate the pages.
+ * For small allocations we'll try contiguous first and then fall back on page by page.
+ */
+#if RTLNX_VER_MIN(2,4,22)
+ if ( fContiguous
+ || cb <= PAGE_SIZE * 2)
+ {
+# ifdef VBOX_USE_INSERT_PAGE
+ paPages = alloc_pages(fFlagsLnx | __GFP_COMP | __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
+# else
+ paPages = alloc_pages(fFlagsLnx | __GFP_NOWARN, rtR0MemObjLinuxOrder(cPages));
+# endif
+ if (paPages)
+ {
+ fContiguous = true;
+ for (iPage = 0; iPage < cPages; iPage++)
+ pMemLnx->apPages[iPage] = &paPages[iPage];
+ }
+ else if (fContiguous)
+ {
+ rtR0MemObjDelete(&pMemLnx->Core);
+ return rcNoMem;
+ }
+ }
+
+ if (!fContiguous)
+ {
+ /** @todo Try use alloc_pages_bulk_array when available, it should be faster
+ * than a alloc_page loop. Put it in #ifdefs similar to
+ * IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC. */
+ for (iPage = 0; iPage < cPages; iPage++)
+ {
+ pMemLnx->apPages[iPage] = alloc_page(fFlagsLnx | __GFP_NOWARN);
+ if (RT_UNLIKELY(!pMemLnx->apPages[iPage]))
+ {
+ while (iPage-- > 0)
+ __free_page(pMemLnx->apPages[iPage]);
+ rtR0MemObjDelete(&pMemLnx->Core);
+ return rcNoMem;
+ }
+ }
+ }
+
+#else /* < 2.4.22 */
+ /** @todo figure out why we didn't allocate page-by-page on 2.4.21 and older... */
+ paPages = alloc_pages(fFlagsLnx, rtR0MemObjLinuxOrder(cPages));
+ if (!paPages)
+ {
+ rtR0MemObjDelete(&pMemLnx->Core);
+ return rcNoMem;
+ }
+ for (iPage = 0; iPage < cPages; iPage++)
+ {
+ pMemLnx->apPages[iPage] = &paPages[iPage];
+ if (fExecutable)
+ MY_SET_PAGES_EXEC(pMemLnx->apPages[iPage], 1);
+ if (PageHighMem(pMemLnx->apPages[iPage]))
+ BUG();
+ }
+
+ fContiguous = true;
+#endif /* < 2.4.22 */
+ pMemLnx->fContiguous = fContiguous;
+ pMemLnx->fExecutable = fExecutable;
+
+#if RTLNX_VER_MAX(4,5,0)
+ /*
+ * Reserve the pages.
+ *
+ * Linux >= 4.5 with CONFIG_DEBUG_VM panics when setting PG_reserved on compound
+ * pages. According to Michal Hocko this shouldn't be necessary anyway because
+ * as pages which are not on the LRU list are never evictable.
+ */
+ for (iPage = 0; iPage < cPages; iPage++)
+ SetPageReserved(pMemLnx->apPages[iPage]);
+#endif
+
+ /*
+ * Note that the physical address of memory allocated with alloc_pages(flags, order)
+ * is always 2^(PAGE_SHIFT+order)-aligned.
+ */
+ if ( fContiguous
+ && uAlignment > PAGE_SIZE)
+ {
+ /*
+ * Check for alignment constraints. The physical address of memory allocated with
+ * alloc_pages(flags, order) is always 2^(PAGE_SHIFT+order)-aligned.
+ */
+ if (RT_UNLIKELY(page_to_phys(pMemLnx->apPages[0]) & (uAlignment - 1)))
+ {
+ /*
+ * This should never happen!
+ */
+ printk("rtR0MemObjLinuxAllocPages(cb=0x%lx, uAlignment=0x%lx): alloc_pages(..., %d) returned physical memory at 0x%lx!\n",
+ (unsigned long)cb, (unsigned long)uAlignment, rtR0MemObjLinuxOrder(cPages), (unsigned long)page_to_phys(pMemLnx->apPages[0]));
+ rtR0MemObjLinuxFreePages(pMemLnx);
+ return rcNoMem;
+ }
+ }
+
+ *ppMemLnx = pMemLnx;
+ return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees the physical pages allocated by the rtR0MemObjLinuxAllocPages() call.
+ *
+ * This method does NOT free the object.
+ *
+ * @param pMemLnx The object which physical pages should be freed.
+ */
+static void rtR0MemObjLinuxFreePages(PRTR0MEMOBJLNX pMemLnx)
+{
+ size_t iPage = pMemLnx->cPages;
+ if (iPage > 0)
+ {
+ /*
+ * Restore the page flags.
+ */
+ while (iPage-- > 0)
+ {
+#if RTLNX_VER_MAX(4,5,0)
+ /* See SetPageReserved() in rtR0MemObjLinuxAllocPages() */
+ ClearPageReserved(pMemLnx->apPages[iPage]);
+#endif
+#if RTLNX_VER_MAX(2,4,22)
+ if (pMemLnx->fExecutable)
+ MY_SET_PAGES_NOEXEC(pMemLnx->apPages[iPage], 1);
+#endif
+ }
+
+ /*
+ * Free the pages.
+ */
+#if RTLNX_VER_MIN(2,4,22)
+ if (!pMemLnx->fContiguous)
+ {
+ iPage = pMemLnx->cPages;
+ while (iPage-- > 0)
+ __free_page(pMemLnx->apPages[iPage]);
+ }
+ else
+#endif
+ __free_pages(pMemLnx->apPages[0], rtR0MemObjLinuxOrder(pMemLnx->cPages));
+
+ pMemLnx->cPages = 0;
+ }
+}
+
+
+#ifdef IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC
+/**
+ * User data passed to the apply_to_page_range() callback.
+ */
+typedef struct LNXAPPLYPGRANGE
+{
+ /** Pointer to the memory object. */
+ PRTR0MEMOBJLNX pMemLnx;
+ /** The page protection flags to apply. */
+ pgprot_t fPg;
+} LNXAPPLYPGRANGE;
+/** Pointer to the user data. */
+typedef LNXAPPLYPGRANGE *PLNXAPPLYPGRANGE;
+/** Pointer to the const user data. */
+typedef const LNXAPPLYPGRANGE *PCLNXAPPLYPGRANGE;
+
+/**
+ * Callback called in apply_to_page_range().
+ *
+ * @returns Linux status code.
+ * @param pPte Pointer to the page table entry for the given address.
+ * @param uAddr The address to apply the new protection to.
+ * @param pvUser The opaque user data.
+ */
+static int rtR0MemObjLinuxApplyPageRange(pte_t *pPte, unsigned long uAddr, void *pvUser)
+{
+ PCLNXAPPLYPGRANGE pArgs = (PCLNXAPPLYPGRANGE)pvUser;
+ PRTR0MEMOBJLNX pMemLnx = pArgs->pMemLnx;
+ size_t idxPg = (uAddr - (unsigned long)pMemLnx->Core.pv) >> PAGE_SHIFT;
+
+ set_pte(pPte, mk_pte(pMemLnx->apPages[idxPg], pArgs->fPg));
+ return 0;
+}
+#endif
+
+
+/**
+ * Maps the allocation into ring-0.
+ *
+ * This will update the RTR0MEMOBJLNX::Core.pv and RTR0MEMOBJ::fMappedToRing0 members.
+ *
+ * Contiguous mappings that isn't in 'high' memory will already be mapped into kernel
+ * space, so we'll use that mapping if possible. If execute access is required, we'll
+ * play safe and do our own mapping.
+ *
+ * @returns IPRT status code.
+ * @param pMemLnx The linux memory object to map.
+ * @param fExecutable Whether execute access is required.
+ */
+static int rtR0MemObjLinuxVMap(PRTR0MEMOBJLNX pMemLnx, bool fExecutable)
+{
+ int rc = VINF_SUCCESS;
+
+ /*
+ * Choose mapping strategy.
+ */
+ bool fMustMap = fExecutable
+ || !pMemLnx->fContiguous;
+ if (!fMustMap)
+ {
+ size_t iPage = pMemLnx->cPages;
+ while (iPage-- > 0)
+ if (PageHighMem(pMemLnx->apPages[iPage]))
+ {
+ fMustMap = true;
+ break;
+ }
+ }
+
+ Assert(!pMemLnx->Core.pv);
+ Assert(!pMemLnx->fMappedToRing0);
+
+ if (fMustMap)
+ {
+ /*
+ * Use vmap - 2.4.22 and later.
+ */
+#if RTLNX_VER_MIN(2,4,22)
+ pgprot_t fPg;
+ pgprot_val(fPg) = _PAGE_PRESENT | _PAGE_RW;
+# ifdef _PAGE_NX
+ if (!fExecutable)
+ pgprot_val(fPg) |= _PAGE_NX;
+# endif
+
+# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
+ if (fExecutable)
+ {
+# if RTLNX_VER_MIN(3,2,51)
+ pte_t **papPtes = (pte_t **)kmalloc_array(pMemLnx->cPages, sizeof(papPtes[0]), GFP_KERNEL);
+# else
+ pte_t **papPtes = (pte_t **)kmalloc(pMemLnx->cPages * sizeof(papPtes[0]), GFP_KERNEL);
+# endif
+ if (papPtes)
+ {
+ pMemLnx->pArea = alloc_vm_area(pMemLnx->Core.cb, papPtes); /* Note! pArea->nr_pages is not set. */
+ if (pMemLnx->pArea)
+ {
+ size_t i;
+ Assert(pMemLnx->pArea->size >= pMemLnx->Core.cb); /* Note! includes guard page. */
+ Assert(pMemLnx->pArea->addr);
+# ifdef _PAGE_NX
+ pgprot_val(fPg) |= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
+# endif
+ pMemLnx->papPtesForArea = papPtes;
+ for (i = 0; i < pMemLnx->cPages; i++)
+ *papPtes[i] = mk_pte(pMemLnx->apPages[i], fPg);
+ pMemLnx->Core.pv = pMemLnx->pArea->addr;
+ pMemLnx->fMappedToRing0 = true;
+ }
+ else
+ {
+ kfree(papPtes);
+ rc = VERR_MAP_FAILED;
+ }
+ }
+ else
+ rc = VERR_MAP_FAILED;
+ }
+ else
+# endif
+ {
+# if defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
+ if (fExecutable)
+ pgprot_val(fPg) |= _PAGE_NX; /* Uses RTR0MemObjProtect to clear NX when memory ready, W^X fashion. */
+# endif
+
+# ifdef VM_MAP
+ pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_MAP, fPg);
+# else
+ pMemLnx->Core.pv = vmap(&pMemLnx->apPages[0], pMemLnx->cPages, VM_ALLOC, fPg);
+# endif
+ if (pMemLnx->Core.pv)
+ pMemLnx->fMappedToRing0 = true;
+ else
+ rc = VERR_MAP_FAILED;
+ }
+#else /* < 2.4.22 */
+ rc = VERR_NOT_SUPPORTED;
+#endif
+ }
+ else
+ {
+ /*
+ * Use the kernel RAM mapping.
+ */
+ pMemLnx->Core.pv = phys_to_virt(page_to_phys(pMemLnx->apPages[0]));
+ Assert(pMemLnx->Core.pv);
+ }
+
+ return rc;
+}
+
+
+/**
+ * Undoes what rtR0MemObjLinuxVMap() did.
+ *
+ * @param pMemLnx The linux memory object.
+ */
+static void rtR0MemObjLinuxVUnmap(PRTR0MEMOBJLNX pMemLnx)
+{
+#if RTLNX_VER_MIN(2,4,22)
+# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
+ if (pMemLnx->pArea)
+ {
+# if 0
+ pte_t **papPtes = pMemLnx->papPtesForArea;
+ size_t i;
+ for (i = 0; i < pMemLnx->cPages; i++)
+ *papPtes[i] = 0;
+# endif
+ free_vm_area(pMemLnx->pArea);
+ kfree(pMemLnx->papPtesForArea);
+ pMemLnx->pArea = NULL;
+ pMemLnx->papPtesForArea = NULL;
+ }
+ else
+# endif
+ if (pMemLnx->fMappedToRing0)
+ {
+ Assert(pMemLnx->Core.pv);
+ vunmap(pMemLnx->Core.pv);
+ pMemLnx->fMappedToRing0 = false;
+ }
+#else /* < 2.4.22 */
+ Assert(!pMemLnx->fMappedToRing0);
+#endif
+ pMemLnx->Core.pv = NULL;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
+
+ /*
+ * Release any memory that we've allocated or locked.
+ */
+ switch (pMemLnx->Core.enmType)
+ {
+ case RTR0MEMOBJTYPE_PAGE:
+ case RTR0MEMOBJTYPE_LOW:
+ case RTR0MEMOBJTYPE_CONT:
+ case RTR0MEMOBJTYPE_PHYS:
+ case RTR0MEMOBJTYPE_PHYS_NC:
+ rtR0MemObjLinuxVUnmap(pMemLnx);
+ rtR0MemObjLinuxFreePages(pMemLnx);
+ break;
+
+ case RTR0MEMOBJTYPE_LARGE_PAGE:
+ {
+ uint32_t const cLargePages = pMemLnx->Core.cb >> (pMemLnx->cLargePageOrder + PAGE_SHIFT);
+ uint32_t iLargePage;
+ for (iLargePage = 0; iLargePage < cLargePages; iLargePage++)
+ __free_pages(pMemLnx->apPages[iLargePage << pMemLnx->cLargePageOrder], pMemLnx->cLargePageOrder);
+ pMemLnx->cPages = 0;
+
+#ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
+ Assert(!pMemLnx->pArea);
+ Assert(!pMemLnx->papPtesForArea);
+#endif
+ break;
+ }
+
+ case RTR0MEMOBJTYPE_LOCK:
+ if (pMemLnx->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
+ {
+ struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
+ size_t iPage;
+ Assert(pTask);
+ if (pTask && pTask->mm)
+ LNX_MM_DOWN_READ(pTask->mm);
+
+ iPage = pMemLnx->cPages;
+ while (iPage-- > 0)
+ {
+ if (!PageReserved(pMemLnx->apPages[iPage]))
+ SetPageDirty(pMemLnx->apPages[iPage]);
+#if RTLNX_VER_MIN(4,6,0)
+ put_page(pMemLnx->apPages[iPage]);
+#else
+ page_cache_release(pMemLnx->apPages[iPage]);
+#endif
+ }
+
+ if (pTask && pTask->mm)
+ LNX_MM_UP_READ(pTask->mm);
+ }
+ /* else: kernel memory - nothing to do here. */
+ break;
+
+ case RTR0MEMOBJTYPE_RES_VIRT:
+ Assert(pMemLnx->Core.pv);
+ if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
+ {
+ struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
+ Assert(pTask);
+ if (pTask && pTask->mm)
+ rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
+ }
+ else
+ {
+ vunmap(pMemLnx->Core.pv);
+
+ Assert(pMemLnx->cPages == 1 && pMemLnx->apPages[0] != NULL);
+ __free_page(pMemLnx->apPages[0]);
+ pMemLnx->apPages[0] = NULL;
+ pMemLnx->cPages = 0;
+ }
+ pMemLnx->Core.pv = NULL;
+ break;
+
+ case RTR0MEMOBJTYPE_MAPPING:
+ Assert(pMemLnx->cPages == 0); Assert(pMemLnx->Core.pv);
+ if (pMemLnx->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
+ {
+ struct task_struct *pTask = rtR0ProcessToLinuxTask(pMemLnx->Core.u.Lock.R0Process);
+ Assert(pTask);
+ if (pTask && pTask->mm)
+ rtR0MemObjLinuxDoMunmap(pMemLnx->Core.pv, pMemLnx->Core.cb, pTask);
+ }
+ else
+ vunmap(pMemLnx->Core.pv);
+ pMemLnx->Core.pv = NULL;
+ break;
+
+ default:
+ AssertMsgFailed(("enmType=%d\n", pMemLnx->Core.enmType));
+ return VERR_INTERNAL_ERROR;
+ }
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC();
+ return VINF_SUCCESS;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ PRTR0MEMOBJLNX pMemLnx;
+ int rc;
+
+#if RTLNX_VER_MIN(2,4,22)
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_HIGHUSER,
+ false /* non-contiguous */, fExecutable, VERR_NO_MEMORY, pszTag);
+#else
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_PAGE, cb, PAGE_SIZE, GFP_USER,
+ false /* non-contiguous */, fExecutable, VERR_NO_MEMORY, pszTag);
+#endif
+ if (RT_SUCCESS(rc))
+ {
+ rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
+ if (RT_SUCCESS(rc))
+ {
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+ }
+
+ rtR0MemObjLinuxFreePages(pMemLnx);
+ rtR0MemObjDelete(&pMemLnx->Core);
+ }
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags,
+ const char *pszTag)
+{
+#ifdef GFP_TRANSHUGE
+ /*
+ * Allocate a memory object structure that's large enough to contain
+ * the page pointer array.
+ */
+# ifdef __GFP_MOVABLE
+ unsigned const fGfp = (GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE;
+# else
+ unsigned const fGfp = (GFP_TRANSHUGE | __GFP_ZERO);
+# endif
+ size_t const cPagesPerLarge = cbLargePage >> PAGE_SHIFT;
+ unsigned const cLargePageOrder = rtR0MemObjLinuxOrder(cPagesPerLarge);
+ size_t const cLargePages = cb >> (cLargePageOrder + PAGE_SHIFT);
+ size_t const cPages = cb >> PAGE_SHIFT;
+ PRTR0MEMOBJLNX pMemLnx;
+
+ Assert(RT_BIT_64(cLargePageOrder + PAGE_SHIFT) == cbLargePage);
+ pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]),
+ RTR0MEMOBJTYPE_LARGE_PAGE, NULL, cb, pszTag);
+ if (pMemLnx)
+ {
+ size_t iLargePage;
+
+ pMemLnx->Core.fFlags |= RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC;
+ pMemLnx->cLargePageOrder = cLargePageOrder;
+ pMemLnx->cPages = cPages;
+
+ /*
+ * Allocate the requested number of large pages.
+ */
+ for (iLargePage = 0; iLargePage < cLargePages; iLargePage++)
+ {
+ struct page *paPages = alloc_pages(fGfp, cLargePageOrder);
+ if (paPages)
+ {
+ size_t const iPageBase = iLargePage << cLargePageOrder;
+ size_t iPage = cPagesPerLarge;
+ while (iPage-- > 0)
+ pMemLnx->apPages[iPageBase + iPage] = &paPages[iPage];
+ }
+ else
+ {
+ /*Log(("rtR0MemObjNativeAllocLarge: cb=%#zx cPages=%#zx cLargePages=%#zx cLargePageOrder=%u cPagesPerLarge=%#zx iLargePage=%#zx -> failed!\n",
+ cb, cPages, cLargePages, cLargePageOrder, cPagesPerLarge, iLargePage, paPages));*/
+ while (iLargePage-- > 0)
+ __free_pages(pMemLnx->apPages[iLargePage << (cLargePageOrder - PAGE_SHIFT)], cLargePageOrder);
+ rtR0MemObjDelete(&pMemLnx->Core);
+ return VERR_NO_MEMORY;
+ }
+ }
+ *ppMem = &pMemLnx->Core;
+ return VINF_SUCCESS;
+ }
+ return VERR_NO_MEMORY;
+
+#else
+ /*
+ * We don't call rtR0MemObjFallbackAllocLarge here as it can be a really
+ * bad idea to trigger the swap daemon and whatnot. So, just fail.
+ */
+ RT_NOREF(ppMem, cb, cbLargePage, fFlags, pszTag);
+ return VERR_NOT_SUPPORTED;
+#endif
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ PRTR0MEMOBJLNX pMemLnx;
+ int rc;
+
+ /* Try to avoid GFP_DMA. GFM_DMA32 was introduced with Linux 2.6.15. */
+#if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
+ /* ZONE_DMA32: 0-4GB */
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA32,
+ false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY, pszTag);
+ if (RT_FAILURE(rc))
+#endif
+#ifdef RT_ARCH_AMD64
+ /* ZONE_DMA: 0-16MB */
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_DMA,
+ false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY, pszTag);
+#else
+# ifdef CONFIG_X86_PAE
+# endif
+ /* ZONE_NORMAL: 0-896MB */
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_LOW, cb, PAGE_SIZE, GFP_USER,
+ false /* non-contiguous */, fExecutable, VERR_NO_LOW_MEMORY, pszTag);
+#endif
+ if (RT_SUCCESS(rc))
+ {
+ rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
+ if (RT_SUCCESS(rc))
+ {
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+ }
+
+ rtR0MemObjLinuxFreePages(pMemLnx);
+ rtR0MemObjDelete(&pMemLnx->Core);
+ }
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ PRTR0MEMOBJLNX pMemLnx;
+ int rc;
+
+#if (defined(RT_ARCH_AMD64) || defined(CONFIG_X86_PAE)) && defined(GFP_DMA32)
+ /* ZONE_DMA32: 0-4GB */
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA32,
+ true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY, pszTag);
+ if (RT_FAILURE(rc))
+#endif
+#ifdef RT_ARCH_AMD64
+ /* ZONE_DMA: 0-16MB */
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_DMA,
+ true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY, pszTag);
+#else
+ /* ZONE_NORMAL (32-bit hosts): 0-896MB */
+ rc = rtR0MemObjLinuxAllocPages(&pMemLnx, RTR0MEMOBJTYPE_CONT, cb, PAGE_SIZE, GFP_USER,
+ true /* contiguous */, fExecutable, VERR_NO_CONT_MEMORY, pszTag);
+#endif
+ if (RT_SUCCESS(rc))
+ {
+ rc = rtR0MemObjLinuxVMap(pMemLnx, fExecutable);
+ if (RT_SUCCESS(rc))
+ {
+#if defined(RT_STRICT) && (defined(RT_ARCH_AMD64) || defined(CONFIG_HIGHMEM64G))
+ size_t iPage = pMemLnx->cPages;
+ while (iPage-- > 0)
+ Assert(page_to_phys(pMemLnx->apPages[iPage]) < _4G);
+#endif
+ pMemLnx->Core.u.Cont.Phys = page_to_phys(pMemLnx->apPages[0]);
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+ }
+
+ rtR0MemObjLinuxFreePages(pMemLnx);
+ rtR0MemObjDelete(&pMemLnx->Core);
+ }
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+/**
+ * Worker for rtR0MemObjLinuxAllocPhysSub that tries one allocation strategy.
+ *
+ * @returns IPRT status code.
+ * @param ppMemLnx Where to
+ * @param enmType The object type.
+ * @param cb The size of the allocation.
+ * @param uAlignment The alignment of the physical memory.
+ * Only valid for fContiguous == true, ignored otherwise.
+ * @param PhysHighest See rtR0MemObjNativeAllocPhys.
+ * @param pszTag Allocation tag used for statistics and such.
+ * @param fGfp The Linux GFP flags to use for the allocation.
+ */
+static int rtR0MemObjLinuxAllocPhysSub2(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
+ size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, const char *pszTag, gfp_t fGfp)
+{
+ PRTR0MEMOBJLNX pMemLnx;
+ int rc = rtR0MemObjLinuxAllocPages(&pMemLnx, enmType, cb, uAlignment, fGfp,
+ enmType == RTR0MEMOBJTYPE_PHYS /* contiguous / non-contiguous */,
+ false /*fExecutable*/, VERR_NO_PHYS_MEMORY, pszTag);
+ if (RT_FAILURE(rc))
+ return rc;
+
+ /*
+ * Check the addresses if necessary. (Can be optimized a bit for PHYS.)
+ */
+ if (PhysHighest != NIL_RTHCPHYS)
+ {
+ size_t iPage = pMemLnx->cPages;
+ while (iPage-- > 0)
+ if (page_to_phys(pMemLnx->apPages[iPage]) > PhysHighest)
+ {
+ rtR0MemObjLinuxFreePages(pMemLnx);
+ rtR0MemObjDelete(&pMemLnx->Core);
+ return VERR_NO_MEMORY;
+ }
+ }
+
+ /*
+ * Complete the object.
+ */
+ if (enmType == RTR0MEMOBJTYPE_PHYS)
+ {
+ pMemLnx->Core.u.Phys.PhysBase = page_to_phys(pMemLnx->apPages[0]);
+ pMemLnx->Core.u.Phys.fAllocated = true;
+ }
+ *ppMem = &pMemLnx->Core;
+ return rc;
+}
+
+
+/**
+ * Worker for rtR0MemObjNativeAllocPhys and rtR0MemObjNativeAllocPhysNC.
+ *
+ * @returns IPRT status code.
+ * @param ppMem Where to store the memory object pointer on success.
+ * @param enmType The object type.
+ * @param cb The size of the allocation.
+ * @param uAlignment The alignment of the physical memory.
+ * Only valid for enmType == RTR0MEMOBJTYPE_PHYS, ignored otherwise.
+ * @param PhysHighest See rtR0MemObjNativeAllocPhys.
+ * @param pszTag Allocation tag used for statistics and such.
+ */
+static int rtR0MemObjLinuxAllocPhysSub(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJTYPE enmType,
+ size_t cb, size_t uAlignment, RTHCPHYS PhysHighest, const char *pszTag)
+{
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * There are two clear cases and that's the <=16MB and anything-goes ones.
+ * When the physical address limit is somewhere in-between those two we'll
+ * just have to try, starting with HIGHUSER and working our way thru the
+ * different types, hoping we'll get lucky.
+ *
+ * We should probably move this physical address restriction logic up to
+ * the page alloc function as it would be more efficient there. But since
+ * we don't expect this to be a performance issue just yet it can wait.
+ */
+ if (PhysHighest == NIL_RTHCPHYS)
+ /* ZONE_HIGHMEM: the whole physical memory */
+ rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_HIGHUSER);
+ else if (PhysHighest <= _1M * 16)
+ /* ZONE_DMA: 0-16MB */
+ rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_DMA);
+ else
+ {
+ rc = VERR_NO_MEMORY;
+ if (RT_FAILURE(rc))
+ /* ZONE_HIGHMEM: the whole physical memory */
+ rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_HIGHUSER);
+ if (RT_FAILURE(rc))
+ /* ZONE_NORMAL: 0-896MB */
+ rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_USER);
+#ifdef GFP_DMA32
+ if (RT_FAILURE(rc))
+ /* ZONE_DMA32: 0-4GB */
+ rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_DMA32);
+#endif
+ if (RT_FAILURE(rc))
+ /* ZONE_DMA: 0-16MB */
+ rc = rtR0MemObjLinuxAllocPhysSub2(ppMem, enmType, cb, uAlignment, PhysHighest, pszTag, GFP_DMA);
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+/**
+ * Translates a kernel virtual address to a linux page structure by walking the
+ * page tables.
+ *
+ * @note We do assume that the page tables will not change as we are walking
+ * them. This assumption is rather forced by the fact that I could not
+ * immediately see any way of preventing this from happening. So, we
+ * take some extra care when accessing them.
+ *
+ * Because of this, we don't want to use this function on memory where
+ * attribute changes to nearby pages is likely to cause large pages to
+ * be used or split up. So, don't use this for the linear mapping of
+ * physical memory.
+ *
+ * @returns Pointer to the page structur or NULL if it could not be found.
+ * @param pv The kernel virtual address.
+ */
+RTDECL(struct page *) rtR0MemObjLinuxVirtToPage(void *pv)
+{
+ unsigned long ulAddr = (unsigned long)pv;
+ unsigned long pfn;
+ struct page *pPage;
+ pte_t *pEntry;
+ union
+ {
+ pgd_t Global;
+#if RTLNX_VER_MIN(4,12,0)
+ p4d_t Four;
+#endif
+#if RTLNX_VER_MIN(2,6,11)
+ pud_t Upper;
+#endif
+ pmd_t Middle;
+ pte_t Entry;
+ } u;
+
+ /* Should this happen in a situation this code will be called in? And if
+ * so, can it change under our feet? See also
+ * "Documentation/vm/active_mm.txt" in the kernel sources. */
+ if (RT_UNLIKELY(!current->active_mm))
+ return NULL;
+ u.Global = *pgd_offset(current->active_mm, ulAddr);
+ if (RT_UNLIKELY(pgd_none(u.Global)))
+ return NULL;
+#if RTLNX_VER_MIN(2,6,11)
+# if RTLNX_VER_MIN(4,12,0)
+ u.Four = *p4d_offset(&u.Global, ulAddr);
+ if (RT_UNLIKELY(p4d_none(u.Four)))
+ return NULL;
+ if (p4d_large(u.Four))
+ {
+ pPage = p4d_page(u.Four);
+ AssertReturn(pPage, NULL);
+ pfn = page_to_pfn(pPage); /* doing the safe way... */
+ AssertCompile(P4D_SHIFT - PAGE_SHIFT < 31);
+ pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (P4D_SHIFT - PAGE_SHIFT)) - 1);
+ return pfn_to_page(pfn);
+ }
+ u.Upper = *pud_offset(&u.Four, ulAddr);
+# else /* < 4.12 */
+ u.Upper = *pud_offset(&u.Global, ulAddr);
+# endif /* < 4.12 */
+ if (RT_UNLIKELY(pud_none(u.Upper)))
+ return NULL;
+# if RTLNX_VER_MIN(2,6,25)
+ if (pud_large(u.Upper))
+ {
+ pPage = pud_page(u.Upper);
+ AssertReturn(pPage, NULL);
+ pfn = page_to_pfn(pPage); /* doing the safe way... */
+ pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PUD_SHIFT - PAGE_SHIFT)) - 1);
+ return pfn_to_page(pfn);
+ }
+# endif
+ u.Middle = *pmd_offset(&u.Upper, ulAddr);
+#else /* < 2.6.11 */
+ u.Middle = *pmd_offset(&u.Global, ulAddr);
+#endif /* < 2.6.11 */
+ if (RT_UNLIKELY(pmd_none(u.Middle)))
+ return NULL;
+#if RTLNX_VER_MIN(2,6,0)
+ if (pmd_large(u.Middle))
+ {
+ pPage = pmd_page(u.Middle);
+ AssertReturn(pPage, NULL);
+ pfn = page_to_pfn(pPage); /* doing the safe way... */
+ pfn += (ulAddr >> PAGE_SHIFT) & ((UINT32_C(1) << (PMD_SHIFT - PAGE_SHIFT)) - 1);
+ return pfn_to_page(pfn);
+ }
+#endif
+
+#if RTLNX_VER_MIN(6,5,0) || RTLNX_RHEL_RANGE(9,4, 9,99)
+ pEntry = __pte_map(&u.Middle, ulAddr);
+#elif RTLNX_VER_MIN(2,5,5) || defined(pte_offset_map) /* As usual, RHEL 3 had pte_offset_map earlier. */
+ pEntry = pte_offset_map(&u.Middle, ulAddr);
+#else
+ pEntry = pte_offset(&u.Middle, ulAddr);
+#endif
+ if (RT_UNLIKELY(!pEntry))
+ return NULL;
+ u.Entry = *pEntry;
+#if RTLNX_VER_MIN(2,5,5) || defined(pte_offset_map)
+ pte_unmap(pEntry);
+#endif
+
+ if (RT_UNLIKELY(!pte_present(u.Entry)))
+ return NULL;
+ return pte_page(u.Entry);
+}
+RT_EXPORT_SYMBOL(rtR0MemObjLinuxVirtToPage);
+
+
+DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment,
+ const char *pszTag)
+{
+ return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS, cb, uAlignment, PhysHighest, pszTag);
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, const char *pszTag)
+{
+ return rtR0MemObjLinuxAllocPhysSub(ppMem, RTR0MEMOBJTYPE_PHYS_NC, cb, PAGE_SIZE, PhysHighest, pszTag);
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy,
+ const char *pszTag)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * All we need to do here is to validate that we can use
+ * ioremap on the specified address (32/64-bit dma_addr_t).
+ */
+ PRTR0MEMOBJLNX pMemLnx;
+ dma_addr_t PhysAddr = Phys;
+ AssertMsgReturn(PhysAddr == Phys, ("%#llx\n", (unsigned long long)Phys), VERR_ADDRESS_TOO_BIG);
+
+ pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_PHYS, NULL, cb, pszTag);
+ if (!pMemLnx)
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+ }
+
+ pMemLnx->Core.u.Phys.PhysBase = PhysAddr;
+ pMemLnx->Core.u.Phys.fAllocated = false;
+ pMemLnx->Core.u.Phys.uCachePolicy = uCachePolicy;
+ Assert(!pMemLnx->cPages);
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+
+/* openSUSE Leap 42.3 detection :-/ */
+#if RTLNX_VER_RANGE(4,4,0, 4,6,0) && defined(FAULT_FLAG_REMOTE)
+# define GET_USER_PAGES_API KERNEL_VERSION(4, 10, 0) /* no typo! */
+#else
+# define GET_USER_PAGES_API LINUX_VERSION_CODE
+#endif
+
+DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess,
+ RTR0PROCESS R0Process, const char *pszTag)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ const int cPages = cb >> PAGE_SHIFT;
+ struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+ struct vm_area_struct **papVMAs;
+# endif
+ PRTR0MEMOBJLNX pMemLnx;
+ int rc = VERR_NO_MEMORY;
+ int const fWrite = fAccess & RTMEM_PROT_WRITE ? 1 : 0;
+
+ /*
+ * Check for valid task and size overflows.
+ */
+ if (!pTask)
+ return VERR_NOT_SUPPORTED;
+ if (((size_t)cPages << PAGE_SHIFT) != cb)
+ return VERR_OUT_OF_RANGE;
+
+ /*
+ * Allocate the memory object and a temporary buffer for the VMAs.
+ */
+ pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK,
+ (void *)R3Ptr, cb, pszTag);
+ if (!pMemLnx)
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+ }
+
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+ papVMAs = (struct vm_area_struct **)RTMemAlloc(sizeof(*papVMAs) * cPages);
+ if (papVMAs)
+ {
+# endif
+ LNX_MM_DOWN_READ(pTask->mm);
+
+ /*
+ * Get user pages.
+ */
+/** @todo r=bird: Should we not force read access too? */
+#if GET_USER_PAGES_API >= KERNEL_VERSION(4, 6, 0)
+ if (R0Process == RTR0ProcHandleSelf())
+ rc = get_user_pages(R3Ptr, /* Where from. */
+ cPages, /* How many pages. */
+# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
+ fWrite ? FOLL_WRITE | /* Write to memory. */
+ FOLL_FORCE /* force write access. */
+ : 0, /* Write to memory. */
+# else
+ fWrite, /* Write to memory. */
+ fWrite, /* force write access. */
+# endif
+ &pMemLnx->apPages[0] /* Page array. */
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+ , papVMAs /* vmas */
+# endif
+ );
+ /*
+ * Actually this should not happen at the moment as call this function
+ * only for our own process.
+ */
+ else
+ rc = get_user_pages_remote(
+# if GET_USER_PAGES_API < KERNEL_VERSION(5, 9, 0)
+ pTask, /* Task for fault accounting. */
+# endif
+ pTask->mm, /* Whose pages. */
+ R3Ptr, /* Where from. */
+ cPages, /* How many pages. */
+# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 9, 0)
+ fWrite ? FOLL_WRITE | /* Write to memory. */
+ FOLL_FORCE /* force write access. */
+ : 0, /* Write to memory. */
+# else
+ fWrite, /* Write to memory. */
+ fWrite, /* force write access. */
+# endif
+ &pMemLnx->apPages[0] /* Page array. */
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+ , papVMAs /* vmas */
+# endif
+# if GET_USER_PAGES_API >= KERNEL_VERSION(4, 10, 0)
+ , NULL /* locked */
+# endif
+ );
+#else /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
+ rc = get_user_pages(pTask, /* Task for fault accounting. */
+ pTask->mm, /* Whose pages. */
+ R3Ptr, /* Where from. */
+ cPages, /* How many pages. */
+/* The get_user_pages API change was back-ported to 4.4.168. */
+# if RTLNX_VER_RANGE(4,4,168, 4,5,0)
+ fWrite ? FOLL_WRITE | /* Write to memory. */
+ FOLL_FORCE /* force write access. */
+ : 0, /* Write to memory. */
+# else
+ fWrite, /* Write to memory. */
+ fWrite, /* force write access. */
+# endif
+ &pMemLnx->apPages[0] /* Page array. */
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+ , papVMAs /* vmas */
+# endif
+ );
+#endif /* GET_USER_PAGES_API < KERNEL_VERSION(4, 6, 0) */
+ if (rc == cPages)
+ {
+ /*
+ * Flush dcache (required?), protect against fork and _really_ pin the page
+ * table entries. get_user_pages() will protect against swapping out the
+ * pages but it will NOT protect against removing page table entries. This
+ * can be achieved with
+ * - using mlock / mmap(..., MAP_LOCKED, ...) from userland. This requires
+ * an appropriate limit set up with setrlimit(..., RLIMIT_MEMLOCK, ...).
+ * Usual Linux distributions support only a limited size of locked pages
+ * (e.g. 32KB).
+ * - setting the PageReserved bit (as we do in rtR0MemObjLinuxAllocPages()
+ * or by
+ * - setting the VM_LOCKED flag. This is the same as doing mlock() without
+ * a range check.
+ */
+ /** @todo The Linux fork() protection will require more work if this API
+ * is to be used for anything but locking VM pages. */
+ while (rc-- > 0)
+ {
+ flush_dcache_page(pMemLnx->apPages[rc]);
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+# if RTLNX_VER_MIN(6,3,0)
+ vm_flags_set(papVMAs[rc], VM_DONTCOPY | VM_LOCKED);
+# else
+ papVMAs[rc]->vm_flags |= VM_DONTCOPY | VM_LOCKED;
+# endif
+# endif
+ }
+
+ LNX_MM_UP_READ(pTask->mm);
+
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+ RTMemFree(papVMAs);
+# endif
+
+ pMemLnx->Core.u.Lock.R0Process = R0Process;
+ pMemLnx->cPages = cPages;
+ Assert(!pMemLnx->fMappedToRing0);
+ *ppMem = &pMemLnx->Core;
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+
+ /*
+ * Failed - we need to unlock any pages that we succeeded to lock.
+ */
+ while (rc-- > 0)
+ {
+ if (!PageReserved(pMemLnx->apPages[rc]))
+ SetPageDirty(pMemLnx->apPages[rc]);
+#if RTLNX_VER_MIN(4,6,0)
+ put_page(pMemLnx->apPages[rc]);
+#else
+ page_cache_release(pMemLnx->apPages[rc]);
+#endif
+ }
+
+ LNX_MM_UP_READ(pTask->mm);
+
+ rc = VERR_LOCK_FAILED;
+
+# if GET_USER_PAGES_API < KERNEL_VERSION(6, 5, 0)
+ RTMemFree(papVMAs);
+ }
+# endif
+
+ rtR0MemObjDelete(&pMemLnx->Core);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, const char *pszTag)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ void *pvLast = (uint8_t *)pv + cb - 1;
+ size_t const cPages = cb >> PAGE_SHIFT;
+ PRTR0MEMOBJLNX pMemLnx;
+ bool fLinearMapping;
+ int rc;
+ uint8_t *pbPage;
+ size_t iPage;
+ NOREF(fAccess);
+
+ if ( !RTR0MemKernelIsValidAddr(pv)
+ || !RTR0MemKernelIsValidAddr(pv + cb))
+ return VERR_INVALID_PARAMETER;
+
+ /*
+ * The lower part of the kernel memory has a linear mapping between
+ * physical and virtual addresses. So we take a short cut here. This is
+ * assumed to be the cleanest way to handle those addresses (and the code
+ * is well tested, though the test for determining it is not very nice).
+ * If we ever decide it isn't we can still remove it.
+ */
+#if 0
+ fLinearMapping = (unsigned long)pvLast < VMALLOC_START;
+#else
+ fLinearMapping = (unsigned long)pv >= (unsigned long)__va(0)
+ && (unsigned long)pvLast < (unsigned long)high_memory;
+#endif
+
+ /*
+ * Allocate the memory object.
+ */
+ pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(RT_UOFFSETOF_DYN(RTR0MEMOBJLNX, apPages[cPages]), RTR0MEMOBJTYPE_LOCK,
+ pv, cb, pszTag);
+ if (!pMemLnx)
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+ }
+
+ /*
+ * Gather the pages.
+ * We ASSUME all kernel pages are non-swappable and non-movable.
+ */
+ rc = VINF_SUCCESS;
+ pbPage = (uint8_t *)pvLast;
+ iPage = cPages;
+ if (!fLinearMapping)
+ {
+ while (iPage-- > 0)
+ {
+ struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
+ if (RT_UNLIKELY(!pPage))
+ {
+ rc = VERR_LOCK_FAILED;
+ break;
+ }
+ pMemLnx->apPages[iPage] = pPage;
+ pbPage -= PAGE_SIZE;
+ }
+ }
+ else
+ {
+ while (iPage-- > 0)
+ {
+ pMemLnx->apPages[iPage] = virt_to_page(pbPage);
+ pbPage -= PAGE_SIZE;
+ }
+ }
+ if (RT_SUCCESS(rc))
+ {
+ /*
+ * Complete the memory object and return.
+ */
+ pMemLnx->Core.u.Lock.R0Process = NIL_RTR0PROCESS;
+ pMemLnx->cPages = cPages;
+ Assert(!pMemLnx->fMappedToRing0);
+ *ppMem = &pMemLnx->Core;
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+
+ rtR0MemObjDelete(&pMemLnx->Core);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment,
+ const char *pszTag)
+{
+#if RTLNX_VER_MIN(2,4,22)
+ IPRT_LINUX_SAVE_EFL_AC();
+ const size_t cPages = cb >> PAGE_SHIFT;
+ struct page *pDummyPage;
+ struct page **papPages;
+
+ /* check for unsupported stuff. */
+ AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
+ if (uAlignment > PAGE_SIZE)
+ return VERR_NOT_SUPPORTED;
+
+ /*
+ * Allocate a dummy page and create a page pointer array for vmap such that
+ * the dummy page is mapped all over the reserved area.
+ */
+ pDummyPage = alloc_page(GFP_HIGHUSER | __GFP_NOWARN);
+ if (pDummyPage)
+ {
+ papPages = RTMemAlloc(sizeof(*papPages) * cPages);
+ if (papPages)
+ {
+ void *pv;
+ size_t iPage = cPages;
+ while (iPage-- > 0)
+ papPages[iPage] = pDummyPage;
+# ifdef VM_MAP
+ pv = vmap(papPages, cPages, VM_MAP, PAGE_KERNEL_RO);
+# else
+ pv = vmap(papPages, cPages, VM_ALLOC, PAGE_KERNEL_RO);
+# endif
+ RTMemFree(papPages);
+ if (pv)
+ {
+ PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb, pszTag);
+ if (pMemLnx)
+ {
+ pMemLnx->Core.u.ResVirt.R0Process = NIL_RTR0PROCESS;
+ pMemLnx->cPages = 1;
+ pMemLnx->apPages[0] = pDummyPage;
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ vunmap(pv);
+ }
+ }
+ __free_page(pDummyPage);
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+
+#else /* < 2.4.22 */
+ /*
+ * Could probably use ioremap here, but the caller is in a better position than us
+ * to select some safe physical memory.
+ */
+ return VERR_NOT_SUPPORTED;
+#endif
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment,
+ RTR0PROCESS R0Process, const char *pszTag)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ PRTR0MEMOBJLNX pMemLnx;
+ void *pv;
+ struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
+ if (!pTask)
+ return VERR_NOT_SUPPORTED;
+
+ /*
+ * Check that the specified alignment is supported.
+ */
+ if (uAlignment > PAGE_SIZE)
+ return VERR_NOT_SUPPORTED;
+
+ /*
+ * Let rtR0MemObjLinuxDoMmap do the difficult bits.
+ */
+ pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cb, uAlignment, pTask, RTMEM_PROT_NONE);
+ if (pv == (void *)-1)
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+ }
+
+ pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_RES_VIRT, pv, cb, pszTag);
+ if (!pMemLnx)
+ {
+ rtR0MemObjLinuxDoMunmap(pv, cb, pTask);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+ }
+
+ pMemLnx->Core.u.ResVirt.R0Process = R0Process;
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
+ unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag)
+{
+ int rc = VERR_NO_MEMORY;
+ PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
+ PRTR0MEMOBJLNX pMemLnx;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /* Fail if requested to do something we can't. */
+ AssertMsgReturn(pvFixed == (void *)-1, ("%p\n", pvFixed), VERR_NOT_SUPPORTED);
+ if (uAlignment > PAGE_SIZE)
+ return VERR_NOT_SUPPORTED;
+
+ /*
+ * Create the IPRT memory object.
+ */
+ if (!cbSub)
+ cbSub = pMemLnxToMap->Core.cb - offSub;
+ pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub, pszTag);
+ if (pMemLnx)
+ {
+ if (pMemLnxToMap->cPages)
+ {
+#if RTLNX_VER_MIN(2,4,22)
+ /*
+ * Use vmap - 2.4.22 and later.
+ */
+ pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, true /* kernel */);
+ /** @todo We don't really care too much for EXEC here... 5.8 always adds NX. */
+ Assert(((offSub + cbSub) >> PAGE_SHIFT) <= pMemLnxToMap->cPages);
+# ifdef VM_MAP
+ pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_MAP, fPg);
+# else
+ pMemLnx->Core.pv = vmap(&pMemLnxToMap->apPages[offSub >> PAGE_SHIFT], cbSub >> PAGE_SHIFT, VM_ALLOC, fPg);
+# endif
+ if (pMemLnx->Core.pv)
+ {
+ pMemLnx->fMappedToRing0 = true;
+ rc = VINF_SUCCESS;
+ }
+ else
+ rc = VERR_MAP_FAILED;
+
+#else /* < 2.4.22 */
+ /*
+ * Only option here is to share mappings if possible and forget about fProt.
+ */
+ if (rtR0MemObjIsRing3(pMemToMap))
+ rc = VERR_NOT_SUPPORTED;
+ else
+ {
+ rc = VINF_SUCCESS;
+ if (!pMemLnxToMap->Core.pv)
+ rc = rtR0MemObjLinuxVMap(pMemLnxToMap, !!(fProt & RTMEM_PROT_EXEC));
+ if (RT_SUCCESS(rc))
+ {
+ Assert(pMemLnxToMap->Core.pv);
+ pMemLnx->Core.pv = (uint8_t *)pMemLnxToMap->Core.pv + offSub;
+ }
+ }
+#endif
+ }
+ else
+ {
+ /*
+ * MMIO / physical memory.
+ */
+ Assert(pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS && !pMemLnxToMap->Core.u.Phys.fAllocated);
+#if RTLNX_VER_MIN(2,6,25)
+ /*
+ * ioremap() defaults to no caching since the 2.6 kernels.
+ * ioremap_nocache() has been removed finally in 5.6-rc1.
+ */
+ pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
+ ? ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
+ : ioremap_cache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
+#else /* KERNEL_VERSION < 2.6.25 */
+ pMemLnx->Core.pv = pMemLnxToMap->Core.u.Phys.uCachePolicy == RTMEM_CACHE_POLICY_MMIO
+ ? ioremap_nocache(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub)
+ : ioremap(pMemLnxToMap->Core.u.Phys.PhysBase + offSub, cbSub);
+#endif /* KERNEL_VERSION < 2.6.25 */
+ if (pMemLnx->Core.pv)
+ {
+ /** @todo fix protection. */
+ rc = VINF_SUCCESS;
+ }
+ }
+ if (RT_SUCCESS(rc))
+ {
+ pMemLnx->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ rtR0MemObjDelete(&pMemLnx->Core);
+ }
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+#ifdef VBOX_USE_PAE_HACK
+/**
+ * Replace the PFN of a PTE with the address of the actual page.
+ *
+ * The caller maps a reserved dummy page at the address with the desired access
+ * and flags.
+ *
+ * This hack is required for older Linux kernels which don't provide
+ * remap_pfn_range().
+ *
+ * @returns 0 on success, -ENOMEM on failure.
+ * @param mm The memory context.
+ * @param ulAddr The mapping address.
+ * @param Phys The physical address of the page to map.
+ */
+static int rtR0MemObjLinuxFixPte(struct mm_struct *mm, unsigned long ulAddr, RTHCPHYS Phys)
+{
+ int rc = -ENOMEM;
+ pgd_t *pgd;
+
+ spin_lock(&mm->page_table_lock);
+
+ pgd = pgd_offset(mm, ulAddr);
+ if (!pgd_none(*pgd) && !pgd_bad(*pgd))
+ {
+ pmd_t *pmd = pmd_offset(pgd, ulAddr);
+ if (!pmd_none(*pmd))
+ {
+ pte_t *ptep = pte_offset_map(pmd, ulAddr);
+ if (ptep)
+ {
+ pte_t pte = *ptep;
+ pte.pte_high &= 0xfff00000;
+ pte.pte_high |= ((Phys >> 32) & 0x000fffff);
+ pte.pte_low &= 0x00000fff;
+ pte.pte_low |= (Phys & 0xfffff000);
+ set_pte(ptep, pte);
+ pte_unmap(ptep);
+ rc = 0;
+ }
+ }
+ }
+
+ spin_unlock(&mm->page_table_lock);
+ return rc;
+}
+#endif /* VBOX_USE_PAE_HACK */
+
+
+DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
+ unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag)
+{
+ struct task_struct *pTask = rtR0ProcessToLinuxTask(R0Process);
+ PRTR0MEMOBJLNX pMemLnxToMap = (PRTR0MEMOBJLNX)pMemToMap;
+ int rc = VERR_NO_MEMORY;
+ PRTR0MEMOBJLNX pMemLnx;
+#ifdef VBOX_USE_PAE_HACK
+ struct page *pDummyPage;
+ RTHCPHYS DummyPhys;
+#endif
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Check for restrictions.
+ */
+ if (!pTask)
+ return VERR_NOT_SUPPORTED;
+ if (uAlignment > PAGE_SIZE)
+ return VERR_NOT_SUPPORTED;
+
+#ifdef VBOX_USE_PAE_HACK
+ /*
+ * Allocate a dummy page for use when mapping the memory.
+ */
+ pDummyPage = alloc_page(GFP_USER | __GFP_NOWARN);
+ if (!pDummyPage)
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+ }
+ SetPageReserved(pDummyPage);
+ DummyPhys = page_to_phys(pDummyPage);
+#endif
+
+ /*
+ * Create the IPRT memory object.
+ */
+ Assert(!offSub || cbSub);
+ if (cbSub == 0)
+ cbSub = pMemLnxToMap->Core.cb;
+ pMemLnx = (PRTR0MEMOBJLNX)rtR0MemObjNew(sizeof(*pMemLnx), RTR0MEMOBJTYPE_MAPPING, NULL, cbSub, pszTag);
+ if (pMemLnx)
+ {
+ /*
+ * Allocate user space mapping.
+ */
+ void *pv;
+ pv = rtR0MemObjLinuxDoMmap(R3PtrFixed, cbSub, uAlignment, pTask, fProt);
+ if (pv != (void *)-1)
+ {
+ /*
+ * Map page by page into the mmap area.
+ * This is generic, paranoid and not very efficient.
+ */
+ pgprot_t fPg = rtR0MemObjLinuxConvertProt(fProt, false /* user */);
+ unsigned long ulAddrCur = (unsigned long)pv;
+ const size_t cPages = (offSub + cbSub) >> PAGE_SHIFT;
+ size_t iPage;
+
+ LNX_MM_DOWN_WRITE(pTask->mm);
+
+ rc = VINF_SUCCESS;
+ if (pMemLnxToMap->cPages)
+ {
+ for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE)
+ {
+#if RTLNX_VER_MAX(2,6,11)
+ RTHCPHYS Phys = page_to_phys(pMemLnxToMap->apPages[iPage]);
+#endif
+#if RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
+ struct vm_area_struct *vma = find_vma(pTask->mm, ulAddrCur); /* this is probably the same for all the pages... */
+ AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
+#endif
+#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
+ /* remap_page_range() limitation on x86 */
+ AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
+#endif
+
+#if defined(VBOX_USE_INSERT_PAGE) && RTLNX_VER_MIN(2,6,22)
+ rc = vm_insert_page(vma, ulAddrCur, pMemLnxToMap->apPages[iPage]);
+ /* Thes flags help making 100% sure some bad stuff wont happen (swap, core, ++).
+ * See remap_pfn_range() in mm/memory.c */
+
+#if RTLNX_VER_MIN(6,3,0)
+ vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
+#elif RTLNX_VER_MIN(3,7,0)
+ vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
+#else
+ vma->vm_flags |= VM_RESERVED;
+#endif
+#elif RTLNX_VER_MIN(2,6,11)
+ rc = remap_pfn_range(vma, ulAddrCur, page_to_pfn(pMemLnxToMap->apPages[iPage]), PAGE_SIZE, fPg);
+#elif defined(VBOX_USE_PAE_HACK)
+ rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
+ if (!rc)
+ rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
+#elif RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
+ rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
+#else /* 2.4 */
+ rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
+#endif
+ if (rc)
+ {
+ rc = VERR_NO_MEMORY;
+ break;
+ }
+ }
+ }
+ else
+ {
+ RTHCPHYS Phys;
+ if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_PHYS)
+ Phys = pMemLnxToMap->Core.u.Phys.PhysBase;
+ else if (pMemLnxToMap->Core.enmType == RTR0MEMOBJTYPE_CONT)
+ Phys = pMemLnxToMap->Core.u.Cont.Phys;
+ else
+ {
+ AssertMsgFailed(("%d\n", pMemLnxToMap->Core.enmType));
+ Phys = NIL_RTHCPHYS;
+ }
+ if (Phys != NIL_RTHCPHYS)
+ {
+ for (iPage = offSub >> PAGE_SHIFT; iPage < cPages; iPage++, ulAddrCur += PAGE_SIZE, Phys += PAGE_SIZE)
+ {
+#if RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
+ struct vm_area_struct *vma = find_vma(pTask->mm, ulAddrCur); /* this is probably the same for all the pages... */
+ AssertBreakStmt(vma, rc = VERR_INTERNAL_ERROR);
+#endif
+#if RTLNX_VER_MAX(2,6,0) && defined(RT_ARCH_X86)
+ /* remap_page_range() limitation on x86 */
+ AssertBreakStmt(Phys < _4G, rc = VERR_NO_MEMORY);
+#endif
+
+#if RTLNX_VER_MIN(2,6,11)
+ rc = remap_pfn_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
+#elif defined(VBOX_USE_PAE_HACK)
+ rc = remap_page_range(vma, ulAddrCur, DummyPhys, PAGE_SIZE, fPg);
+ if (!rc)
+ rc = rtR0MemObjLinuxFixPte(pTask->mm, ulAddrCur, Phys);
+#elif RTLNX_VER_MIN(2,6,0) || defined(HAVE_26_STYLE_REMAP_PAGE_RANGE)
+ rc = remap_page_range(vma, ulAddrCur, Phys, PAGE_SIZE, fPg);
+#else /* 2.4 */
+ rc = remap_page_range(ulAddrCur, Phys, PAGE_SIZE, fPg);
+#endif
+ if (rc)
+ {
+ rc = VERR_NO_MEMORY;
+ break;
+ }
+ }
+ }
+ }
+
+#ifdef CONFIG_NUMA_BALANCING
+# if RTLNX_VER_MAX(3,13,0) && RTLNX_RHEL_MAX(7,0)
+# define VBOX_NUMA_HACK_OLD
+# endif
+ if (RT_SUCCESS(rc))
+ {
+ /** @todo Ugly hack! But right now we have no other means to
+ * disable automatic NUMA page balancing. */
+# ifdef RT_OS_X86
+# ifdef VBOX_NUMA_HACK_OLD
+ pTask->mm->numa_next_reset = jiffies + 0x7fffffffUL;
+# endif
+ pTask->mm->numa_next_scan = jiffies + 0x7fffffffUL;
+# else
+# ifdef VBOX_NUMA_HACK_OLD
+ pTask->mm->numa_next_reset = jiffies + 0x7fffffffffffffffUL;
+# endif
+ pTask->mm->numa_next_scan = jiffies + 0x7fffffffffffffffUL;
+# endif
+ }
+#endif /* CONFIG_NUMA_BALANCING */
+
+ LNX_MM_UP_WRITE(pTask->mm);
+
+ if (RT_SUCCESS(rc))
+ {
+#ifdef VBOX_USE_PAE_HACK
+ __free_page(pDummyPage);
+#endif
+ pMemLnx->Core.pv = pv;
+ pMemLnx->Core.u.Mapping.R0Process = R0Process;
+ *ppMem = &pMemLnx->Core;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+
+ /*
+ * Bail out.
+ */
+ rtR0MemObjLinuxDoMunmap(pv, cbSub, pTask);
+ }
+ rtR0MemObjDelete(&pMemLnx->Core);
+ }
+#ifdef VBOX_USE_PAE_HACK
+ __free_page(pDummyPage);
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+
+
+DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
+{
+# ifdef IPRT_USE_ALLOC_VM_AREA_FOR_EXEC
+ /*
+ * Currently only supported when we've got addresses PTEs from the kernel.
+ */
+ PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
+ if (pMemLnx->pArea && pMemLnx->papPtesForArea)
+ {
+ pgprot_t const fPg = rtR0MemObjLinuxConvertProt(fProt, true /*fKernel*/);
+ size_t const cPages = (offSub + cbSub) >> PAGE_SHIFT;
+ pte_t **papPtes = pMemLnx->papPtesForArea;
+ size_t i;
+
+ for (i = offSub >> PAGE_SHIFT; i < cPages; i++)
+ {
+ set_pte(papPtes[i], mk_pte(pMemLnx->apPages[i], fPg));
+ }
+ preempt_disable();
+ __flush_tlb_all();
+ preempt_enable();
+ return VINF_SUCCESS;
+ }
+# elif defined(IPRT_USE_APPLY_TO_PAGE_RANGE_FOR_EXEC)
+ PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
+ if ( pMemLnx->fExecutable
+ && pMemLnx->fMappedToRing0)
+ {
+ LNXAPPLYPGRANGE Args;
+ Args.pMemLnx = pMemLnx;
+ Args.fPg = rtR0MemObjLinuxConvertProt(fProt, true /*fKernel*/);
+ int rcLnx = apply_to_page_range(current->active_mm, (unsigned long)pMemLnx->Core.pv + offSub, cbSub,
+ rtR0MemObjLinuxApplyPageRange, (void *)&Args);
+ if (rcLnx)
+ return VERR_NOT_SUPPORTED;
+
+ return VINF_SUCCESS;
+ }
+# endif
+
+ NOREF(pMem);
+ NOREF(offSub);
+ NOREF(cbSub);
+ NOREF(fProt);
+ return VERR_NOT_SUPPORTED;
+}
+
+
+DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
+{
+ PRTR0MEMOBJLNX pMemLnx = (PRTR0MEMOBJLNX)pMem;
+
+ if (pMemLnx->cPages)
+ return page_to_phys(pMemLnx->apPages[iPage]);
+
+ switch (pMemLnx->Core.enmType)
+ {
+ case RTR0MEMOBJTYPE_CONT:
+ return pMemLnx->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);
+
+ case RTR0MEMOBJTYPE_PHYS:
+ return pMemLnx->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);
+
+ /* the parent knows */
+ case RTR0MEMOBJTYPE_MAPPING:
+ return rtR0MemObjNativeGetPagePhysAddr(pMemLnx->Core.uRel.Child.pParent, iPage);
+
+ /* cPages > 0 */
+ case RTR0MEMOBJTYPE_LOW:
+ case RTR0MEMOBJTYPE_LOCK:
+ case RTR0MEMOBJTYPE_PHYS_NC:
+ case RTR0MEMOBJTYPE_PAGE:
+ case RTR0MEMOBJTYPE_LARGE_PAGE:
+ default:
+ AssertMsgFailed(("%d\n", pMemLnx->Core.enmType));
+ RT_FALL_THROUGH();
+
+ case RTR0MEMOBJTYPE_RES_VIRT:
+ return NIL_RTHCPHYS;
+ }
+}
+
diff --git a/src/VBox/Runtime/r0drv/linux/memuserkernel-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/memuserkernel-r0drv-linux.c
new file mode 100644
index 00000000..e526132d
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/memuserkernel-r0drv-linux.c
@@ -0,0 +1,191 @@
+/* $Id: memuserkernel-r0drv-linux.c $ */
+/** @file
+ * IPRT - User & Kernel Memory, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2009-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/mem.h>
+#include <iprt/errcore.h>
+
+
+RTR0DECL(int) RTR0MemUserCopyFrom(void *pvDst, RTR3PTR R3PtrSrc, size_t cb)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ if (RT_LIKELY(copy_from_user(pvDst, (void *)R3PtrSrc, cb) == 0))
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_ACCESS_DENIED;
+}
+RT_EXPORT_SYMBOL(RTR0MemUserCopyFrom);
+
+
+RTR0DECL(int) RTR0MemUserCopyTo(RTR3PTR R3PtrDst, void const *pvSrc, size_t cb)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ if (RT_LIKELY(copy_to_user((void *)R3PtrDst, pvSrc, cb) == 0))
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_ACCESS_DENIED;
+}
+RT_EXPORT_SYMBOL(RTR0MemUserCopyTo);
+
+
+RTR0DECL(bool) RTR0MemUserIsValidAddr(RTR3PTR R3Ptr)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+#if RTLNX_VER_MIN(5,0,0) || RTLNX_RHEL_MIN(8,1)
+ bool fRc = access_ok((void *)R3Ptr, 1);
+#else
+ bool fRc = access_ok(VERIFY_READ, (void *)R3Ptr, 1);
+#endif
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return fRc;
+}
+RT_EXPORT_SYMBOL(RTR0MemUserIsValidAddr);
+
+
+RTR0DECL(bool) RTR0MemKernelIsValidAddr(void *pv)
+{
+ /* Couldn't find a straight forward way of doing this... */
+#if defined(RT_ARCH_X86) && defined(CONFIG_X86_HIGH_ENTRY)
+ return true; /* ?? */
+#elif defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
+ return (uintptr_t)pv >= PAGE_OFFSET;
+#else
+# error "PORT ME"
+#if RTLNX_VER_MIN(5,0,0) || RTLNX_RHEL_MIN(8,1)
+ return !access_ok(pv, 1);
+#else
+ return !access_ok(VERIFY_READ, pv, 1);
+#endif
+#endif
+}
+RT_EXPORT_SYMBOL(RTR0MemKernelIsValidAddr);
+
+
+RTR0DECL(bool) RTR0MemAreKrnlAndUsrDifferent(void)
+{
+#if defined(RT_ARCH_X86) && defined(CONFIG_X86_HIGH_ENTRY) /* ?? */
+ return false;
+#else
+ return true;
+#endif
+}
+RT_EXPORT_SYMBOL(RTR0MemAreKrnlAndUsrDifferent);
+
+
+/**
+ * Treats both source and destination as unsafe buffers.
+ */
+static int rtR0MemKernelCopyLnxWorker(void *pvDst, void const *pvSrc, size_t cb)
+{
+#if RTLNX_VER_MIN(2,5,55)
+/* _ASM_EXTABLE was introduced in 2.6.25 from what I can tell. Using #ifndef
+ here since it has to be a macro and you never know what someone might have
+ backported to an earlier kernel release. */
+# ifndef _ASM_EXTABLE
+# if ARCH_BITS == 32
+# define _ASM_EXTABLE(a_Instr, a_Resume) \
+ ".section __ex_table,\"a\"\n" \
+ ".balign 4\n" \
+ ".long " #a_Instr "\n" \
+ ".long " #a_Resume "\n" \
+ ".previous\n"
+# else
+# define _ASM_EXTABLE(a_Instr, a_Resume) \
+ ".section __ex_table,\"a\"\n" \
+ ".balign 8\n" \
+ ".quad " #a_Instr "\n" \
+ ".quad " #a_Resume "\n" \
+ ".previous\n"
+# endif
+# endif /* !_ASM_EXTABLE */
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC(); /* paranoia */
+ if (!cb)
+ return VINF_SUCCESS;
+
+ __asm__ __volatile__ ("cld\n"
+ "1:\n\t"
+ "rep; movsb\n"
+ "2:\n\t"
+ ".section .fixup,\"ax\"\n"
+ "3:\n\t"
+ "movl %4, %0\n\t"
+ "jmp 2b\n\t"
+ ".previous\n"
+ _ASM_EXTABLE(1b, 3b)
+ : "=r" (rc),
+ "=D" (pvDst),
+ "=S" (pvSrc),
+ "=c" (cb)
+ : "i" (VERR_ACCESS_DENIED),
+ "0" (VINF_SUCCESS),
+ "1" (pvDst),
+ "2" (pvSrc),
+ "3" (cb)
+ : "memory");
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+#else
+ return VERR_NOT_SUPPORTED;
+#endif
+}
+
+
+RTR0DECL(int) RTR0MemKernelCopyFrom(void *pvDst, void const *pvSrc, size_t cb)
+{
+ return rtR0MemKernelCopyLnxWorker(pvDst, pvSrc, cb);
+}
+RT_EXPORT_SYMBOL(RTR0MemKernelCopyFrom);
+
+
+RTR0DECL(int) RTR0MemKernelCopyTo(void *pvDst, void const *pvSrc, size_t cb)
+{
+ return rtR0MemKernelCopyLnxWorker(pvDst, pvSrc, cb);
+}
+RT_EXPORT_SYMBOL(RTR0MemKernelCopyTo);
+
diff --git a/src/VBox/Runtime/r0drv/linux/mp-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/mp-r0drv-linux.c
new file mode 100644
index 00000000..5ac05db5
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/mp-r0drv-linux.c
@@ -0,0 +1,640 @@
+/* $Id: mp-r0drv-linux.c $ */
+/** @file
+ * IPRT - Multiprocessor, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2008-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/mp.h>
+#include <iprt/cpuset.h>
+#include <iprt/err.h>
+#include <iprt/asm.h>
+#include <iprt/thread.h>
+#include "r0drv/mp-r0drv.h"
+
+
+/*********************************************************************************************************************************
+* Defined Constants And Macros *
+*********************************************************************************************************************************/
+#if defined(nr_cpumask_bits) || RTLNX_VER_MIN(2,6,28)
+# define VBOX_NR_CPUMASK_BITS (nr_cpumask_bits) /* same as nr_cpu_ids */
+#else
+# define VBOX_NR_CPUMASK_BITS (NR_CPUS)
+#endif
+
+
+RTDECL(RTCPUID) RTMpCpuId(void)
+{
+ return smp_processor_id();
+}
+RT_EXPORT_SYMBOL(RTMpCpuId);
+
+
+RTDECL(int) RTMpCurSetIndex(void)
+{
+ return smp_processor_id();
+}
+RT_EXPORT_SYMBOL(RTMpCurSetIndex);
+
+
+RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
+{
+ return *pidCpu = smp_processor_id();
+}
+RT_EXPORT_SYMBOL(RTMpCurSetIndexAndId);
+
+
+RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
+{
+ return idCpu < RTCPUSET_MAX_CPUS && idCpu < VBOX_NR_CPUMASK_BITS ? (int)idCpu : -1;
+}
+RT_EXPORT_SYMBOL(RTMpCpuIdToSetIndex);
+
+
+RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
+{
+ return (unsigned)iCpu < VBOX_NR_CPUMASK_BITS ? (RTCPUID)iCpu : NIL_RTCPUID;
+}
+RT_EXPORT_SYMBOL(RTMpCpuIdFromSetIndex);
+
+
+RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
+{
+ return VBOX_NR_CPUMASK_BITS - 1;
+}
+RT_EXPORT_SYMBOL(RTMpGetMaxCpuId);
+
+
+RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
+{
+#if defined(CONFIG_SMP)
+# if RTLNX_VER_MIN(2,6,2) || defined(cpu_possible)
+ return idCpu < VBOX_NR_CPUMASK_BITS && cpu_possible(idCpu);
+# else /* < 2.5.29 */
+ return idCpu < (RTCPUID)(smp_num_cpus);
+# endif
+#else
+ return idCpu == RTMpCpuId();
+#endif
+}
+RT_EXPORT_SYMBOL(RTMpIsCpuPossible);
+
+
+RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
+{
+ RTCPUID idCpu;
+
+ RTCpuSetEmpty(pSet);
+ idCpu = RTMpGetMaxCpuId();
+ do
+ {
+ if (RTMpIsCpuPossible(idCpu))
+ RTCpuSetAdd(pSet, idCpu);
+ } while (idCpu-- > 0);
+ return pSet;
+}
+RT_EXPORT_SYMBOL(RTMpGetSet);
+
+
+RTDECL(RTCPUID) RTMpGetCount(void)
+{
+#ifdef CONFIG_SMP
+# if RTLNX_VER_MIN(2,6,4) || defined(num_possible_cpus)
+ return num_possible_cpus();
+# elif RTLNX_VER_MAX(2,5,0)
+ return smp_num_cpus;
+# else
+ RTCPUSET Set;
+ RTMpGetSet(&Set);
+ return RTCpuSetCount(&Set);
+# endif
+#else
+ return 1;
+#endif
+}
+RT_EXPORT_SYMBOL(RTMpGetCount);
+
+
+RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
+{
+#ifdef CONFIG_SMP
+# if RTLNX_VER_MIN(2,6,0) || defined(cpu_online)
+ return idCpu < VBOX_NR_CPUMASK_BITS && cpu_online(idCpu);
+# else /* 2.4: */
+ return idCpu < VBOX_NR_CPUMASK_BITS && cpu_online_map & RT_BIT_64(idCpu);
+# endif
+#else
+ return idCpu == RTMpCpuId();
+#endif
+}
+RT_EXPORT_SYMBOL(RTMpIsCpuOnline);
+
+
+RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
+{
+#ifdef CONFIG_SMP
+ RTCPUID idCpu;
+
+ RTCpuSetEmpty(pSet);
+ idCpu = RTMpGetMaxCpuId();
+ do
+ {
+ if (RTMpIsCpuOnline(idCpu))
+ RTCpuSetAdd(pSet, idCpu);
+ } while (idCpu-- > 0);
+#else
+ RTCpuSetEmpty(pSet);
+ RTCpuSetAdd(pSet, RTMpCpuId());
+#endif
+ return pSet;
+}
+RT_EXPORT_SYMBOL(RTMpGetOnlineSet);
+
+
+RTDECL(RTCPUID) RTMpGetOnlineCount(void)
+{
+#ifdef CONFIG_SMP
+# if RTLNX_VER_MIN(2,6,0) || defined(num_online_cpus)
+ return num_online_cpus();
+# else
+ RTCPUSET Set;
+ RTMpGetOnlineSet(&Set);
+ return RTCpuSetCount(&Set);
+# endif
+#else
+ return 1;
+#endif
+}
+RT_EXPORT_SYMBOL(RTMpGetOnlineCount);
+
+
+RTDECL(bool) RTMpIsCpuWorkPending(void)
+{
+ /** @todo (not used on non-Windows platforms yet). */
+ return false;
+}
+RT_EXPORT_SYMBOL(RTMpIsCpuWorkPending);
+
+
+/**
+ * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER.
+ *
+ * @param pvInfo Pointer to the RTMPARGS package.
+ */
+static void rtmpLinuxWrapper(void *pvInfo)
+{
+ PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
+ ASMAtomicIncU32(&pArgs->cHits);
+ pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
+}
+
+#ifdef CONFIG_SMP
+
+# if RTLNX_VER_MIN(2,6,27)
+/**
+ * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER, does hit
+ * increment after calling the worker.
+ *
+ * @param pvInfo Pointer to the RTMPARGS package.
+ */
+static void rtmpLinuxWrapperPostInc(void *pvInfo)
+{
+ PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
+ pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
+ ASMAtomicIncU32(&pArgs->cHits);
+}
+# endif
+
+
+/**
+ * Wrapper between the native linux all-cpu callbacks and PFNRTWORKER.
+ *
+ * @param pvInfo Pointer to the RTMPARGS package.
+ */
+static void rtmpLinuxAllWrapper(void *pvInfo)
+{
+ PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
+ PRTCPUSET pWorkerSet = pArgs->pWorkerSet;
+ RTCPUID idCpu = RTMpCpuId();
+ Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+ if (RTCpuSetIsMember(pWorkerSet, idCpu))
+ {
+ pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
+ RTCpuSetDel(pWorkerSet, idCpu);
+ }
+}
+
+#endif /* CONFIG_SMP */
+
+RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ RTMPARGS Args;
+ RTCPUSET OnlineSet;
+ RTCPUID idCpu;
+#ifdef CONFIG_SMP
+ uint32_t cLoops;
+#endif
+
+ RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+
+ Args.pfnWorker = pfnWorker;
+ Args.pvUser1 = pvUser1;
+ Args.pvUser2 = pvUser2;
+ Args.idCpu = NIL_RTCPUID;
+ Args.cHits = 0;
+
+ RTThreadPreemptDisable(&PreemptState);
+ RTMpGetOnlineSet(&OnlineSet);
+ Args.pWorkerSet = &OnlineSet;
+ idCpu = RTMpCpuId();
+
+#ifdef CONFIG_SMP
+ if (RTCpuSetCount(&OnlineSet) > 1)
+ {
+ /* Fire the function on all other CPUs without waiting for completion. */
+# if RTLNX_VER_MIN(5,3,0)
+ smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
+# elif RTLNX_VER_MIN(2,6,27)
+ int rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
+ Assert(!rc); NOREF(rc);
+# else
+ int rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* retry */, 0 /* wait */);
+ Assert(!rc); NOREF(rc);
+# endif
+ }
+#endif
+
+ /* Fire the function on this CPU. */
+ Args.pfnWorker(idCpu, Args.pvUser1, Args.pvUser2);
+ RTCpuSetDel(Args.pWorkerSet, idCpu);
+
+#ifdef CONFIG_SMP
+ /* Wait for all of them finish. */
+ cLoops = 64000;
+ while (!RTCpuSetIsEmpty(Args.pWorkerSet))
+ {
+ /* Periodically check if any CPU in the wait set has gone offline, if so update the wait set. */
+ if (!cLoops--)
+ {
+ RTCPUSET OnlineSetNow;
+ RTMpGetOnlineSet(&OnlineSetNow);
+ RTCpuSetAnd(Args.pWorkerSet, &OnlineSetNow);
+
+ cLoops = 64000;
+ }
+
+ ASMNopPause();
+ }
+#endif
+
+ RTThreadPreemptRestore(&PreemptState);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTMpOnAll);
+
+
+RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
+{
+#ifdef CONFIG_SMP
+ IPRT_LINUX_SAVE_EFL_AC();
+ RTMPARGS Args;
+
+ RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+ Args.pfnWorker = pfnWorker;
+ Args.pvUser1 = pvUser1;
+ Args.pvUser2 = pvUser2;
+ Args.idCpu = NIL_RTCPUID;
+ Args.cHits = 0;
+
+ RTThreadPreemptDisable(&PreemptState);
+# if RTLNX_VER_MIN(5,3,0)
+ smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
+# elif RTLNX_VER_MIN(2,6,27)
+ int rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
+ Assert(rc == 0); NOREF(rc);
+# else /* older kernels */
+ int rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
+ Assert(rc == 0); NOREF(rc);
+# endif /* older kernels */
+ RTThreadPreemptRestore(&PreemptState);
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+#else
+ RT_NOREF(pfnWorker, pvUser1, pvUser2);
+#endif
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTMpOnOthers);
+
+
+#if RTLNX_VER_MAX(2,6,27) && defined(CONFIG_SMP)
+/**
+ * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
+ * employed by RTMpOnPair on older kernels that lacks smp_call_function_many.
+ *
+ * @param pvInfo Pointer to the RTMPARGS package.
+ */
+static void rtMpLinuxOnPairWrapper(void *pvInfo)
+{
+ PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
+ RTCPUID idCpu = RTMpCpuId();
+
+ if ( idCpu == pArgs->idCpu
+ || idCpu == pArgs->idCpu2)
+ {
+ pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
+ ASMAtomicIncU32(&pArgs->cHits);
+ }
+}
+#endif
+
+
+RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
+{
+#ifdef CONFIG_SMP
+ IPRT_LINUX_SAVE_EFL_AC();
+ int rc;
+ RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+# if RTLNX_VER_MIN(2,6,28) /* 2.6.28 introduces CONFIG_CPUMASK_OFFSTACK */
+ cpumask_var_t DstCpuMask;
+# elif RTLNX_VER_MIN(2,6,27)
+ cpumask_t DstCpuMask;
+# endif
+
+ AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
+ AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
+
+ /*
+ * Prepare the CPU mask before we disable preemption.
+ */
+# if RTLNX_VER_MIN(2,6,30)
+ if (!zalloc_cpumask_var(&DstCpuMask, GFP_KERNEL))
+ return VERR_NO_MEMORY;
+ cpumask_set_cpu(idCpu1, DstCpuMask);
+ cpumask_set_cpu(idCpu2, DstCpuMask);
+# elif RTLNX_VER_MIN(2,6,28)
+ if (!alloc_cpumask_var(&DstCpuMask, GFP_KERNEL))
+ return VERR_NO_MEMORY;
+ cpumask_clear(DstCpuMask);
+ cpumask_set_cpu(idCpu1, DstCpuMask);
+ cpumask_set_cpu(idCpu2, DstCpuMask);
+# elif RTLNX_VER_MIN(2,6,27)
+ cpus_clear(DstCpuMask);
+ cpu_set(idCpu1, DstCpuMask);
+ cpu_set(idCpu2, DstCpuMask);
+# endif
+
+ /*
+ * Check that both CPUs are online before doing the broadcast call.
+ */
+ RTThreadPreemptDisable(&PreemptState);
+ if ( RTMpIsCpuOnline(idCpu1)
+ && RTMpIsCpuOnline(idCpu2))
+ {
+ /*
+ * Use the smp_call_function variant taking a cpu mask where available,
+ * falling back on broadcast with filter. Slight snag if one of the
+ * CPUs is the one we're running on, we must do the call and the post
+ * call wait ourselves.
+ */
+ RTCPUID idCpuSelf = RTMpCpuId();
+ bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
+ RTMPARGS Args;
+ Args.pfnWorker = pfnWorker;
+ Args.pvUser1 = pvUser1;
+ Args.pvUser2 = pvUser2;
+ Args.idCpu = idCpu1;
+ Args.idCpu2 = idCpu2;
+ Args.cHits = 0;
+
+# if RTLNX_VER_MIN(2,6,28)
+ smp_call_function_many(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
+ rc = 0;
+# elif RTLNX_VER_MIN(2,6,27)
+ rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
+# else /* older kernels */
+ rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
+# endif /* older kernels */
+ Assert(rc == 0);
+
+ /* Call ourselves if necessary and wait for the other party to be done. */
+ if (fCallSelf)
+ {
+ uint32_t cLoops = 0;
+ rtmpLinuxWrapper(&Args);
+ while (ASMAtomicReadU32(&Args.cHits) < 2)
+ {
+ if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
+ break;
+ cLoops++;
+ ASMNopPause();
+ }
+ }
+
+ Assert(Args.cHits <= 2);
+ if (Args.cHits == 2)
+ rc = VINF_SUCCESS;
+ else if (Args.cHits == 1)
+ rc = VERR_NOT_ALL_CPUS_SHOWED;
+ else if (Args.cHits == 0)
+ rc = VERR_CPU_OFFLINE;
+ else
+ rc = VERR_CPU_IPE_1;
+ }
+ /*
+ * A CPU must be present to be considered just offline.
+ */
+ else if ( RTMpIsCpuPresent(idCpu1)
+ && RTMpIsCpuPresent(idCpu2))
+ rc = VERR_CPU_OFFLINE;
+ else
+ rc = VERR_CPU_NOT_FOUND;
+
+ RTThreadPreemptRestore(&PreemptState);;
+# if RTLNX_VER_MIN(2,6,28)
+ free_cpumask_var(DstCpuMask);
+# endif
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+
+#else /* !CONFIG_SMP */
+ RT_NOREF(idCpu1, idCpu2, fFlags, pfnWorker, pvUser1, pvUser2);
+ return VERR_CPU_NOT_FOUND;
+#endif /* !CONFIG_SMP */
+}
+RT_EXPORT_SYMBOL(RTMpOnPair);
+
+
+RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
+{
+ return true;
+}
+RT_EXPORT_SYMBOL(RTMpOnPairIsConcurrentExecSupported);
+
+
+#if RTLNX_VER_MAX(2,6,19) && defined(CONFIG_SMP)
+/**
+ * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
+ * employed by RTMpOnSpecific on older kernels that lacks smp_call_function_single.
+ *
+ * @param pvInfo Pointer to the RTMPARGS package.
+ */
+static void rtmpOnSpecificLinuxWrapper(void *pvInfo)
+{
+ PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
+ RTCPUID idCpu = RTMpCpuId();
+
+ if (idCpu == pArgs->idCpu)
+ {
+ pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
+ ASMAtomicIncU32(&pArgs->cHits);
+ }
+}
+#endif
+
+
+RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ int rc;
+ RTMPARGS Args;
+
+ RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+ Args.pfnWorker = pfnWorker;
+ Args.pvUser1 = pvUser1;
+ Args.pvUser2 = pvUser2;
+ Args.idCpu = idCpu;
+ Args.cHits = 0;
+
+ if (!RTMpIsCpuPossible(idCpu))
+ return VERR_CPU_NOT_FOUND;
+
+ RTThreadPreemptDisable(&PreemptState);
+ if (idCpu != RTMpCpuId())
+ {
+#ifdef CONFIG_SMP
+ if (RTMpIsCpuOnline(idCpu))
+ {
+# if RTLNX_VER_MIN(2,6,27)
+ rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 1 /* wait */);
+# elif RTLNX_VER_MIN(2,6,19)
+ rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
+# else /* older kernels */
+ rc = smp_call_function(rtmpOnSpecificLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
+# endif /* older kernels */
+ Assert(rc == 0);
+ rc = Args.cHits ? VINF_SUCCESS : VERR_CPU_OFFLINE;
+ }
+ else
+#endif /* CONFIG_SMP */
+ rc = VERR_CPU_OFFLINE;
+ }
+ else
+ {
+ rtmpLinuxWrapper(&Args);
+ rc = VINF_SUCCESS;
+ }
+ RTThreadPreemptRestore(&PreemptState);;
+
+ NOREF(rc);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+RT_EXPORT_SYMBOL(RTMpOnSpecific);
+
+
+#if RTLNX_VER_MIN(2,6,19) && defined(CONFIG_SMP)
+/**
+ * Dummy callback used by RTMpPokeCpu.
+ *
+ * @param pvInfo Ignored.
+ */
+static void rtmpLinuxPokeCpuCallback(void *pvInfo)
+{
+ NOREF(pvInfo);
+}
+#endif
+
+
+RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
+{
+#if RTLNX_VER_MIN(2,6,19)
+ IPRT_LINUX_SAVE_EFL_AC();
+ int rc;
+ if (RTMpIsCpuPossible(idCpu))
+ {
+ if (RTMpIsCpuOnline(idCpu))
+ {
+# ifdef CONFIG_SMP
+# if RTLNX_VER_MIN(2,6,27)
+ rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* wait */);
+# elif RTLNX_VER_MIN(2,6,19)
+ rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* retry */, 0 /* wait */);
+# else /* older kernels */
+# error oops
+# endif /* older kernels */
+ Assert(rc == 0);
+# endif /* CONFIG_SMP */
+ rc = VINF_SUCCESS;
+ }
+ else
+ rc = VERR_CPU_OFFLINE;
+ }
+ else
+ rc = VERR_CPU_NOT_FOUND;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+
+#else /* older kernels */
+ /* no unicast here? */
+ return VERR_NOT_SUPPORTED;
+#endif /* older kernels */
+}
+RT_EXPORT_SYMBOL(RTMpPokeCpu);
+
+
+RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
+{
+ return true;
+}
+RT_EXPORT_SYMBOL(RTMpOnAllIsConcurrentSafe);
+
diff --git a/src/VBox/Runtime/r0drv/linux/mpnotification-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/mpnotification-r0drv-linux.c
new file mode 100644
index 00000000..99b8748e
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/mpnotification-r0drv-linux.c
@@ -0,0 +1,258 @@
+/* $Id: mpnotification-r0drv-linux.c $ */
+/** @file
+ * IPRT - Multiprocessor Event Notifications, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2008-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/errcore.h>
+#include <iprt/cpuset.h>
+#include <iprt/thread.h>
+#include "r0drv/mp-r0drv.h"
+
+#if RTLNX_VER_MIN(4,10,0)
+
+static enum cpuhp_state g_rtR0MpOnline;
+
+/*
+ * Linux 4.10 completely removed CPU notifiers. So let's switch to CPU hotplug
+ * notification.
+ */
+
+static int rtR0MpNotificationLinuxOnline(unsigned int cpu)
+{
+ RTCPUID idCpu = RTMpCpuIdFromSetIndex(cpu);
+ rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, idCpu);
+ return 0;
+}
+
+static int rtR0MpNotificationLinuxOffline(unsigned int cpu)
+{
+ RTCPUID idCpu = RTMpCpuIdFromSetIndex(cpu);
+ rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, idCpu);
+ return 0;
+}
+
+DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
+{
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC();
+ rc = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "vboxdrv:online",
+ rtR0MpNotificationLinuxOnline, rtR0MpNotificationLinuxOffline);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ /*
+ * cpuhp_setup_state_nocalls() returns a positive state number for
+ * CPUHP_AP_ONLINE_DYN or -ENOSPC if there is no free slot available
+ * (see cpuhp_reserve_state / definition of CPUHP_AP_ONLINE_DYN).
+ */
+ AssertMsgReturn(rc > 0, ("%d\n", rc), RTErrConvertFromErrno(rc));
+ g_rtR0MpOnline = rc;
+ return VINF_SUCCESS;
+}
+
+
+DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ cpuhp_remove_state_nocalls(g_rtR0MpOnline);
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
+#elif RTLNX_VER_MIN(2,5,71) && defined(CONFIG_SMP)
+
+static int rtMpNotificationLinuxCallback(struct notifier_block *pNotifierBlock, unsigned long ulNativeEvent, void *pvCpu);
+
+/**
+ * The notifier block we use for registering the callback.
+ */
+static struct notifier_block g_NotifierBlock =
+{
+ .notifier_call = rtMpNotificationLinuxCallback,
+ .next = NULL,
+ .priority = 0
+};
+
+# ifdef CPU_DOWN_FAILED
+/**
+ * The set of CPUs we've seen going offline recently.
+ */
+static RTCPUSET g_MpPendingOfflineSet;
+# endif
+
+
+/**
+ * The native callback.
+ *
+ * @returns NOTIFY_DONE.
+ * @param pNotifierBlock Pointer to g_NotifierBlock.
+ * @param ulNativeEvent The native event.
+ * @param pvCpu The cpu id cast into a pointer value.
+ *
+ * @remarks This can fire with preemption enabled and on any CPU.
+ */
+static int rtMpNotificationLinuxCallback(struct notifier_block *pNotifierBlock, unsigned long ulNativeEvent, void *pvCpu)
+{
+ bool fProcessEvent = false;
+ RTCPUID idCpu = (uintptr_t)pvCpu;
+ NOREF(pNotifierBlock);
+
+ /*
+ * Note that redhat/CentOS ported _some_ of the FROZEN macros
+ * back to their 2.6.18-92.1.10.el5 kernel but actually don't
+ * use them. Thus we have to test for both CPU_TASKS_FROZEN and
+ * the individual event variants.
+ */
+ switch (ulNativeEvent)
+ {
+ /*
+ * Pick up online events or failures to go offline.
+ * Ignore failure events for CPUs we didn't see go offline.
+ */
+# ifdef CPU_DOWN_FAILED
+ case CPU_DOWN_FAILED:
+# if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_FAILED_FROZEN)
+ case CPU_DOWN_FAILED_FROZEN:
+# endif
+ if (!RTCpuSetIsMember(&g_MpPendingOfflineSet, idCpu))
+ break; /* fProcessEvents = false */
+ /* fall thru */
+# endif
+ case CPU_ONLINE:
+# if defined(CPU_TASKS_FROZEN) && defined(CPU_ONLINE_FROZEN)
+ case CPU_ONLINE_FROZEN:
+# endif
+# ifdef CPU_DOWN_FAILED
+ RTCpuSetDel(&g_MpPendingOfflineSet, idCpu);
+# endif
+ fProcessEvent = true;
+ break;
+
+ /*
+ * Pick the earliest possible offline event.
+ * The only important thing here is that we get the event and that
+ * it's exactly one.
+ */
+# ifdef CPU_DOWN_PREPARE
+ case CPU_DOWN_PREPARE:
+# if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_PREPARE_FROZEN)
+ case CPU_DOWN_PREPARE_FROZEN:
+# endif
+ fProcessEvent = true;
+# else
+ case CPU_DEAD:
+# if defined(CPU_TASKS_FROZEN) && defined(CPU_DEAD_FROZEN)
+ case CPU_DEAD_FROZEN:
+# endif
+ /* Don't process CPU_DEAD notifications. */
+# endif
+# ifdef CPU_DOWN_FAILED
+ RTCpuSetAdd(&g_MpPendingOfflineSet, idCpu);
+# endif
+ break;
+ }
+
+ if (!fProcessEvent)
+ return NOTIFY_DONE;
+
+ switch (ulNativeEvent)
+ {
+# ifdef CPU_DOWN_FAILED
+ case CPU_DOWN_FAILED:
+# if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_FAILED_FROZEN)
+ case CPU_DOWN_FAILED_FROZEN:
+# endif
+# endif
+ case CPU_ONLINE:
+# if defined(CPU_TASKS_FROZEN) && defined(CPU_ONLINE_FROZEN)
+ case CPU_ONLINE_FROZEN:
+# endif
+ rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, idCpu);
+ break;
+
+# ifdef CPU_DOWN_PREPARE
+ case CPU_DOWN_PREPARE:
+# if defined(CPU_TASKS_FROZEN) && defined(CPU_DOWN_PREPARE_FROZEN)
+ case CPU_DOWN_PREPARE_FROZEN:
+# endif
+ rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, idCpu);
+ break;
+# endif
+ }
+
+ return NOTIFY_DONE;
+}
+
+
+DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
+{
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+# ifdef CPU_DOWN_FAILED
+ RTCpuSetEmpty(&g_MpPendingOfflineSet);
+# endif
+
+ rc = register_cpu_notifier(&g_NotifierBlock);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ AssertMsgReturn(!rc, ("%d\n", rc), RTErrConvertFromErrno(rc));
+ return VINF_SUCCESS;
+}
+
+
+DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ unregister_cpu_notifier(&g_NotifierBlock);
+ IPRT_LINUX_RESTORE_EFL_AC();
+}
+
+#else /* Not supported / Not needed */
+
+DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
+{
+ return VINF_SUCCESS;
+}
+
+DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
+{
+}
+
+#endif /* Not supported / Not needed */
+
diff --git a/src/VBox/Runtime/r0drv/linux/process-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/process-r0drv-linux.c
new file mode 100644
index 00000000..3179a95e
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/process-r0drv-linux.c
@@ -0,0 +1,59 @@
+/* $Id: process-r0drv-linux.c $ */
+/** @file
+ * IPRT - Process, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/process.h>
+
+
+RTDECL(RTPROCESS) RTProcSelf(void)
+{
+ return (RTPROCESS)current->tgid;
+}
+RT_EXPORT_SYMBOL(RTProcSelf);
+
+
+RTR0DECL(RTR0PROCESS) RTR0ProcHandleSelf(void)
+{
+ return (RTR0PROCESS)current->tgid;
+}
+RT_EXPORT_SYMBOL(RTR0ProcHandleSelf);
+
diff --git a/src/VBox/Runtime/r0drv/linux/rtStrFormatKernelAddress-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/rtStrFormatKernelAddress-r0drv-linux.c
new file mode 100644
index 00000000..14cfc4f7
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/rtStrFormatKernelAddress-r0drv-linux.c
@@ -0,0 +1,66 @@
+/* $Id: rtStrFormatKernelAddress-r0drv-linux.c $ */
+/** @file
+ * IPRT - IPRT String Formatter, ring-0 addresses.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#define LOG_GROUP RTLOGGROUP_STRING
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+#include "internal/string.h"
+
+
+DECLHIDDEN(size_t) rtStrFormatKernelAddress(char *pszBuf, size_t cbBuf, RTR0INTPTR uPtr, signed int cchWidth,
+ signed int cchPrecision, unsigned int fFlags)
+{
+#if !defined(DEBUG) && RTLNX_VER_MIN(2,6,38)
+ RT_NOREF(cchWidth, cchPrecision);
+ /* use the Linux kernel function which is able to handle "%pK" */
+ static const char s_szFmt[] = "0x%pK";
+ const char *pszFmt = s_szFmt;
+ if (!(fFlags & RTSTR_F_SPECIAL))
+ pszFmt += 2;
+ return scnprintf(pszBuf, cbBuf, pszFmt, uPtr);
+#else
+ Assert(cbBuf >= 64);
+ return RTStrFormatNumber(pszBuf, uPtr, 16, cchWidth, cchPrecision, fFlags);
+#endif
+}
diff --git a/src/VBox/Runtime/r0drv/linux/semevent-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/semevent-r0drv-linux.c
new file mode 100644
index 00000000..0b1ca618
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/semevent-r0drv-linux.c
@@ -0,0 +1,296 @@
+/* $Id: semevent-r0drv-linux.c $ */
+/** @file
+ * IPRT - Single Release Event Semaphores, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#define RTSEMEVENT_WITHOUT_REMAPPING
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/semaphore.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/err.h>
+#include <iprt/lockvalidator.h>
+#include <iprt/mem.h>
+
+#include "waitqueue-r0drv-linux.h"
+#include "internal/magics.h"
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+/**
+ * Linux event semaphore.
+ */
+typedef struct RTSEMEVENTINTERNAL
+{
+ /** Magic value (RTSEMEVENT_MAGIC). */
+ uint32_t volatile u32Magic;
+ /** The object status - !0 when signaled and 0 when reset. */
+ uint32_t volatile fState;
+ /** Reference counter. */
+ uint32_t volatile cRefs;
+ /** The wait queue. */
+ wait_queue_head_t Head;
+} RTSEMEVENTINTERNAL, *PRTSEMEVENTINTERNAL;
+
+
+
+RTDECL(int) RTSemEventCreate(PRTSEMEVENT phEventSem)
+{
+ return RTSemEventCreateEx(phEventSem, 0 /*fFlags*/, NIL_RTLOCKVALCLASS, NULL);
+}
+
+
+RTDECL(int) RTSemEventCreateEx(PRTSEMEVENT phEventSem, uint32_t fFlags, RTLOCKVALCLASS hClass, const char *pszNameFmt, ...)
+{
+ PRTSEMEVENTINTERNAL pThis;
+ IPRT_LINUX_SAVE_EFL_AC();
+ RT_NOREF_PV(hClass); RT_NOREF_PV(pszNameFmt);
+
+ AssertReturn(!(fFlags & ~(RTSEMEVENT_FLAGS_NO_LOCK_VAL | RTSEMEVENT_FLAGS_BOOTSTRAP_HACK)), VERR_INVALID_PARAMETER);
+ Assert(!(fFlags & RTSEMEVENT_FLAGS_BOOTSTRAP_HACK) || (fFlags & RTSEMEVENT_FLAGS_NO_LOCK_VAL));
+
+ pThis = (PRTSEMEVENTINTERNAL)RTMemAlloc(sizeof(*pThis));
+ if (!pThis)
+ return VERR_NO_MEMORY;
+
+ pThis->u32Magic = RTSEMEVENT_MAGIC;
+ pThis->fState = 0;
+ pThis->cRefs = 1;
+ init_waitqueue_head(&pThis->Head);
+
+ *phEventSem = pThis;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemEventCreate);
+
+
+/**
+ * Retains a reference to the event semaphore.
+ *
+ * @param pThis The event semaphore.
+ */
+DECLINLINE(void) rtR0SemEventLnxRetain(PRTSEMEVENTINTERNAL pThis)
+{
+ uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
+ Assert(cRefs < 100000); NOREF(cRefs);
+}
+
+
+/**
+ * Releases a reference to the event semaphore.
+ *
+ * @param pThis The event semaphore.
+ */
+DECLINLINE(void) rtR0SemEventLnxRelease(PRTSEMEVENTINTERNAL pThis)
+{
+ if (RT_UNLIKELY(ASMAtomicDecU32(&pThis->cRefs) == 0))
+ RTMemFree(pThis);
+}
+
+
+RTDECL(int) RTSemEventDestroy(RTSEMEVENT hEventSem)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate input.
+ */
+ PRTSEMEVENTINTERNAL pThis = hEventSem;
+ if (pThis == NIL_RTSEMEVENT)
+ return VINF_SUCCESS;
+ AssertMsgReturn(pThis->u32Magic == RTSEMEVENT_MAGIC, ("pThis->u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+ Assert(pThis->cRefs > 0);
+
+ /*
+ * Invalidate it and signal the object just in case.
+ */
+ ASMAtomicWriteU32(&pThis->u32Magic, ~RTSEMEVENT_MAGIC);
+ ASMAtomicWriteU32(&pThis->fState, 0);
+ Assert(!waitqueue_active(&pThis->Head));
+ wake_up_all(&pThis->Head);
+ rtR0SemEventLnxRelease(pThis);
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemEventDestroy);
+
+
+RTDECL(int) RTSemEventSignal(RTSEMEVENT hEventSem)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate input.
+ */
+ PRTSEMEVENTINTERNAL pThis = (PRTSEMEVENTINTERNAL)hEventSem;
+ AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+ AssertMsgReturn(pThis->u32Magic == RTSEMEVENT_MAGIC, ("pThis->u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+ rtR0SemEventLnxRetain(pThis);
+
+ /*
+ * Signal the event object.
+ */
+ ASMAtomicWriteU32(&pThis->fState, 1);
+ wake_up(&pThis->Head);
+
+ rtR0SemEventLnxRelease(pThis);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemEventSignal);
+
+
+/**
+ * Worker for RTSemEventWaitEx and RTSemEventWaitExDebug.
+ *
+ * @returns VBox status code.
+ * @param pThis The event semaphore.
+ * @param fFlags See RTSemEventWaitEx.
+ * @param uTimeout See RTSemEventWaitEx.
+ * @param pSrcPos The source code position of the wait.
+ */
+static int rtR0SemEventLnxWait(PRTSEMEVENTINTERNAL pThis, uint32_t fFlags, uint64_t uTimeout,
+ PCRTLOCKVALSRCPOS pSrcPos)
+{
+ int rc;
+ RT_NOREF_PV(pSrcPos);
+
+ /*
+ * Validate the input.
+ */
+ AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
+ AssertMsgReturn(pThis->u32Magic == RTSEMEVENT_MAGIC, ("%p u32Magic=%RX32\n", pThis, pThis->u32Magic), VERR_INVALID_PARAMETER);
+ AssertReturn(RTSEMWAIT_FLAGS_ARE_VALID(fFlags), VERR_INVALID_PARAMETER);
+ rtR0SemEventLnxRetain(pThis);
+
+ /*
+ * Try grab the event without setting up the wait.
+ */
+ if ( 1 /** @todo check if there are someone waiting already - waitqueue_active, but then what do we do below? */
+ && ASMAtomicCmpXchgU32(&pThis->fState, 0, 1))
+ rc = VINF_SUCCESS;
+ else
+ {
+ /*
+ * We have to wait.
+ */
+ IPRT_LINUX_SAVE_EFL_AC();
+ RTR0SEMLNXWAIT Wait;
+ rc = rtR0SemLnxWaitInit(&Wait, fFlags, uTimeout, &pThis->Head);
+ if (RT_SUCCESS(rc))
+ {
+ IPRT_DEBUG_SEMS_STATE(pThis, 'E');
+ for (;;)
+ {
+ /* The destruction test. */
+ if (RT_UNLIKELY(pThis->u32Magic != RTSEMEVENT_MAGIC))
+ rc = VERR_SEM_DESTROYED;
+ else
+ {
+ rtR0SemLnxWaitPrepare(&Wait);
+
+ /* Check the exit conditions. */
+ if (RT_UNLIKELY(pThis->u32Magic != RTSEMEVENT_MAGIC))
+ rc = VERR_SEM_DESTROYED;
+ else if (ASMAtomicCmpXchgU32(&pThis->fState, 0, 1))
+ rc = VINF_SUCCESS;
+ else if (rtR0SemLnxWaitHasTimedOut(&Wait))
+ rc = VERR_TIMEOUT;
+ else if (rtR0SemLnxWaitWasInterrupted(&Wait))
+ rc = VERR_INTERRUPTED;
+ else
+ {
+ /* Do the wait and then recheck the conditions. */
+ rtR0SemLnxWaitDoIt(&Wait);
+ continue;
+ }
+ }
+ break;
+ }
+
+ rtR0SemLnxWaitDelete(&Wait);
+ IPRT_DEBUG_SEMS_STATE_RC(pThis, 'E', rc);
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ }
+
+ rtR0SemEventLnxRelease(pThis);
+ return rc;
+}
+
+
+RTDECL(int) RTSemEventWaitEx(RTSEMEVENT hEventSem, uint32_t fFlags, uint64_t uTimeout)
+{
+#ifndef RTSEMEVENT_STRICT
+ return rtR0SemEventLnxWait(hEventSem, fFlags, uTimeout, NULL);
+#else
+ RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+ return rtR0SemEventLnxWait(hEventSem, fFlags, uTimeout, &SrcPos);
+#endif
+}
+RT_EXPORT_SYMBOL(RTSemEventWaitEx);
+
+
+RTDECL(int) RTSemEventWaitExDebug(RTSEMEVENT hEventSem, uint32_t fFlags, uint64_t uTimeout,
+ RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+ RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+ return rtR0SemEventLnxWait(hEventSem, fFlags, uTimeout, &SrcPos);
+}
+RT_EXPORT_SYMBOL(RTSemEventWaitExDebug);
+
+
+RTDECL(uint32_t) RTSemEventGetResolution(void)
+{
+ return rtR0SemLnxWaitGetResolution();
+}
+RT_EXPORT_SYMBOL(RTSemEventGetResolution);
+
+
+RTR0DECL(bool) RTSemEventIsSignalSafe(void)
+{
+ return true;
+}
+RT_EXPORT_SYMBOL(RTSemEventIsSignalSafe);
+
diff --git a/src/VBox/Runtime/r0drv/linux/semeventmulti-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/semeventmulti-r0drv-linux.c
new file mode 100644
index 00000000..fbc028a1
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/semeventmulti-r0drv-linux.c
@@ -0,0 +1,361 @@
+/* $Id: semeventmulti-r0drv-linux.c $ */
+/** @file
+ * IPRT - Multiple Release Event Semaphores, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#define RTSEMEVENTMULTI_WITHOUT_REMAPPING
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/semaphore.h>
+
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/err.h>
+#include <iprt/mem.h>
+#include <iprt/lockvalidator.h>
+
+#include "waitqueue-r0drv-linux.h"
+#include "internal/magics.h"
+
+
+/*********************************************************************************************************************************
+* Defined Constants And Macros *
+*********************************************************************************************************************************/
+/** @name fStateAndGen values
+ * @{ */
+/** The state bit number. */
+#define RTSEMEVENTMULTILNX_STATE_BIT 0
+/** The state mask. */
+#define RTSEMEVENTMULTILNX_STATE_MASK RT_BIT_32(RTSEMEVENTMULTILNX_STATE_BIT)
+/** The generation mask. */
+#define RTSEMEVENTMULTILNX_GEN_MASK ~RTSEMEVENTMULTILNX_STATE_MASK
+/** The generation shift. */
+#define RTSEMEVENTMULTILNX_GEN_SHIFT 1
+/** The initial variable value. */
+#define RTSEMEVENTMULTILNX_STATE_GEN_INIT UINT32_C(0xfffffffc)
+/** @} */
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+/**
+ * Linux event semaphore.
+ */
+typedef struct RTSEMEVENTMULTIINTERNAL
+{
+ /** Magic value (RTSEMEVENTMULTI_MAGIC). */
+ uint32_t volatile u32Magic;
+ /** The object state bit and generation counter.
+ * The generation counter is incremented every time the object is
+ * signalled. */
+ uint32_t volatile fStateAndGen;
+ /** Reference counter. */
+ uint32_t volatile cRefs;
+ /** The wait queue. */
+ wait_queue_head_t Head;
+} RTSEMEVENTMULTIINTERNAL, *PRTSEMEVENTMULTIINTERNAL;
+
+
+
+
+
+RTDECL(int) RTSemEventMultiCreate(PRTSEMEVENTMULTI phEventMultiSem)
+{
+ return RTSemEventMultiCreateEx(phEventMultiSem, 0 /*fFlags*/, NIL_RTLOCKVALCLASS, NULL);
+}
+
+
+RTDECL(int) RTSemEventMultiCreateEx(PRTSEMEVENTMULTI phEventMultiSem, uint32_t fFlags, RTLOCKVALCLASS hClass,
+ const char *pszNameFmt, ...)
+{
+ PRTSEMEVENTMULTIINTERNAL pThis;
+ IPRT_LINUX_SAVE_EFL_AC();
+ RT_NOREF_PV(hClass); RT_NOREF_PV(pszNameFmt);
+
+ AssertReturn(!(fFlags & ~RTSEMEVENTMULTI_FLAGS_NO_LOCK_VAL), VERR_INVALID_PARAMETER);
+ pThis = (PRTSEMEVENTMULTIINTERNAL)RTMemAlloc(sizeof(*pThis));
+ if (pThis)
+ {
+ pThis->u32Magic = RTSEMEVENTMULTI_MAGIC;
+ pThis->fStateAndGen = RTSEMEVENTMULTILNX_STATE_GEN_INIT;
+ pThis->cRefs = 1;
+ init_waitqueue_head(&pThis->Head);
+
+ *phEventMultiSem = pThis;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiCreate);
+
+
+/**
+ * Retain a reference to the semaphore.
+ *
+ * @param pThis The semaphore.
+ */
+DECLINLINE(void) rtR0SemEventMultiLnxRetain(PRTSEMEVENTMULTIINTERNAL pThis)
+{
+ uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
+ NOREF(cRefs);
+ Assert(cRefs && cRefs < 100000);
+}
+
+
+/**
+ * Release a reference, destroy the thing if necessary.
+ *
+ * @param pThis The semaphore.
+ */
+DECLINLINE(void) rtR0SemEventMultiLnxRelease(PRTSEMEVENTMULTIINTERNAL pThis)
+{
+ if (RT_UNLIKELY(ASMAtomicDecU32(&pThis->cRefs) == 0))
+ {
+ Assert(pThis->u32Magic != RTSEMEVENTMULTI_MAGIC);
+ RTMemFree(pThis);
+ }
+}
+
+
+RTDECL(int) RTSemEventMultiDestroy(RTSEMEVENTMULTI hEventMultiSem)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate input.
+ */
+ PRTSEMEVENTMULTIINTERNAL pThis = (PRTSEMEVENTMULTIINTERNAL)hEventMultiSem;
+ if (pThis == NIL_RTSEMEVENTMULTI)
+ return VINF_SUCCESS;
+ AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
+ AssertMsgReturn(pThis->u32Magic == RTSEMEVENTMULTI_MAGIC, ("%p u32Magic=%RX32\n", pThis, pThis->u32Magic), VERR_INVALID_PARAMETER);
+ Assert(pThis->cRefs > 0);
+
+ /*
+ * Invalidate it and signal the object just in case.
+ */
+ ASMAtomicWriteU32(&pThis->u32Magic, ~RTSEMEVENTMULTI_MAGIC);
+ ASMAtomicAndU32(&pThis->fStateAndGen, RTSEMEVENTMULTILNX_GEN_MASK);
+ Assert(!waitqueue_active(&pThis->Head));
+ wake_up_all(&pThis->Head);
+ rtR0SemEventMultiLnxRelease(pThis);
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiDestroy);
+
+
+RTDECL(int) RTSemEventMultiSignal(RTSEMEVENTMULTI hEventMultiSem)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ uint32_t fNew;
+ uint32_t fOld;
+
+ /*
+ * Validate input.
+ */
+ PRTSEMEVENTMULTIINTERNAL pThis = (PRTSEMEVENTMULTIINTERNAL)hEventMultiSem;
+ if (!pThis)
+ return VERR_INVALID_PARAMETER;
+ AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
+ AssertMsgReturn(pThis->u32Magic == RTSEMEVENTMULTI_MAGIC, ("%p u32Magic=%RX32\n", pThis, pThis->u32Magic), VERR_INVALID_PARAMETER);
+ rtR0SemEventMultiLnxRetain(pThis);
+
+ /*
+ * Signal the event object. The cause of the paranoia here is racing to try
+ * deal with racing RTSemEventMultiSignal calls (should probably be
+ * forbidden, but it's relatively easy to handle).
+ */
+ do
+ {
+ fNew = fOld = ASMAtomicUoReadU32(&pThis->fStateAndGen);
+ fNew += 1 << RTSEMEVENTMULTILNX_GEN_SHIFT;
+ fNew |= RTSEMEVENTMULTILNX_STATE_MASK;
+ }
+ while (!ASMAtomicCmpXchgU32(&pThis->fStateAndGen, fNew, fOld));
+
+ wake_up_all(&pThis->Head);
+
+ rtR0SemEventMultiLnxRelease(pThis);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiSignal);
+
+
+RTDECL(int) RTSemEventMultiReset(RTSEMEVENTMULTI hEventMultiSem)
+{
+ /*
+ * Validate input.
+ */
+ PRTSEMEVENTMULTIINTERNAL pThis = (PRTSEMEVENTMULTIINTERNAL)hEventMultiSem;
+ if (!pThis)
+ return VERR_INVALID_PARAMETER;
+ AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
+ AssertMsgReturn(pThis->u32Magic == RTSEMEVENTMULTI_MAGIC, ("%p u32Magic=%RX32\n", pThis, pThis->u32Magic), VERR_INVALID_PARAMETER);
+ rtR0SemEventMultiLnxRetain(pThis);
+
+ /*
+ * Reset it.
+ */
+ ASMAtomicAndU32(&pThis->fStateAndGen, ~RTSEMEVENTMULTILNX_STATE_MASK);
+
+ rtR0SemEventMultiLnxRelease(pThis);
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiReset);
+
+
+/**
+ * Worker for RTSemEventMultiWaitEx and RTSemEventMultiWaitExDebug.
+ *
+ * @returns VBox status code.
+ * @param pThis The event semaphore.
+ * @param fFlags See RTSemEventMultiWaitEx.
+ * @param uTimeout See RTSemEventMultiWaitEx.
+ * @param pSrcPos The source code position of the wait.
+ */
+static int rtR0SemEventMultiLnxWait(PRTSEMEVENTMULTIINTERNAL pThis, uint32_t fFlags, uint64_t uTimeout,
+ PCRTLOCKVALSRCPOS pSrcPos)
+{
+ uint32_t fOrgStateAndGen;
+ int rc;
+ RT_NOREF_PV(pSrcPos);
+
+ /*
+ * Validate the input.
+ */
+ AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
+ AssertMsgReturn(pThis->u32Magic == RTSEMEVENTMULTI_MAGIC, ("%p u32Magic=%RX32\n", pThis, pThis->u32Magic), VERR_INVALID_PARAMETER);
+ AssertReturn(RTSEMWAIT_FLAGS_ARE_VALID(fFlags), VERR_INVALID_PARAMETER);
+ rtR0SemEventMultiLnxRetain(pThis);
+
+ /*
+ * Is the event already signalled or do we have to wait?
+ */
+ fOrgStateAndGen = ASMAtomicUoReadU32(&pThis->fStateAndGen);
+ if (fOrgStateAndGen & RTSEMEVENTMULTILNX_STATE_MASK)
+ rc = VINF_SUCCESS;
+ else
+ {
+ /*
+ * We have to wait.
+ */
+ RTR0SEMLNXWAIT Wait;
+ IPRT_LINUX_SAVE_EFL_AC();
+ rc = rtR0SemLnxWaitInit(&Wait, fFlags, uTimeout, &pThis->Head);
+ if (RT_SUCCESS(rc))
+ {
+ IPRT_DEBUG_SEMS_STATE(pThis, 'E');
+ for (;;)
+ {
+ /* The destruction test. */
+ if (RT_UNLIKELY(pThis->u32Magic != RTSEMEVENTMULTI_MAGIC))
+ rc = VERR_SEM_DESTROYED;
+ else
+ {
+ rtR0SemLnxWaitPrepare(&Wait);
+
+ /* Check the exit conditions. */
+ if (RT_UNLIKELY(pThis->u32Magic != RTSEMEVENTMULTI_MAGIC))
+ rc = VERR_SEM_DESTROYED;
+ else if (ASMAtomicUoReadU32(&pThis->fStateAndGen) != fOrgStateAndGen)
+ rc = VINF_SUCCESS;
+ else if (rtR0SemLnxWaitHasTimedOut(&Wait))
+ rc = VERR_TIMEOUT;
+ else if (rtR0SemLnxWaitWasInterrupted(&Wait))
+ rc = VERR_INTERRUPTED;
+ else
+ {
+ /* Do the wait and then recheck the conditions. */
+ rtR0SemLnxWaitDoIt(&Wait);
+ continue;
+ }
+ }
+ break;
+ }
+
+ rtR0SemLnxWaitDelete(&Wait);
+ IPRT_DEBUG_SEMS_STATE_RC(pThis, 'E', rc);
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ }
+
+ rtR0SemEventMultiLnxRelease(pThis);
+ return rc;
+}
+
+
+RTDECL(int) RTSemEventMultiWaitEx(RTSEMEVENTMULTI hEventMultiSem, uint32_t fFlags, uint64_t uTimeout)
+{
+#ifndef RTSEMEVENT_STRICT
+ return rtR0SemEventMultiLnxWait(hEventMultiSem, fFlags, uTimeout, NULL);
+#else
+ RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+ return rtR0SemEventMultiLnxWait(hEventMultiSem, fFlags, uTimeout, &SrcPos);
+#endif
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiWaitEx);
+
+
+RTDECL(int) RTSemEventMultiWaitExDebug(RTSEMEVENTMULTI hEventMultiSem, uint32_t fFlags, uint64_t uTimeout,
+ RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+ RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+ return rtR0SemEventMultiLnxWait(hEventMultiSem, fFlags, uTimeout, &SrcPos);
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiWaitExDebug);
+
+
+RTDECL(uint32_t) RTSemEventMultiGetResolution(void)
+{
+ return rtR0SemLnxWaitGetResolution();
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiGetResolution);
+
+
+RTR0DECL(bool) RTSemEventMultiIsSignalSafe(void)
+{
+ return true;
+}
+RT_EXPORT_SYMBOL(RTSemEventMultiIsSignalSafe);
+
diff --git a/src/VBox/Runtime/r0drv/linux/semfastmutex-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/semfastmutex-r0drv-linux.c
new file mode 100644
index 00000000..0e2dff35
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/semfastmutex-r0drv-linux.c
@@ -0,0 +1,167 @@
+/* $Id: semfastmutex-r0drv-linux.c $ */
+/** @file
+ * IPRT - Fast Mutex Semaphores, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/semaphore.h>
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/errcore.h>
+#if defined(RT_STRICT) || defined(IPRT_DEBUG_SEMS)
+# include <iprt/thread.h>
+#endif
+
+#include "internal/magics.h"
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+/**
+ * Wrapper for the linux semaphore structure.
+ */
+typedef struct RTSEMFASTMUTEXINTERNAL
+{
+ /** Magic value (RTSEMFASTMUTEX_MAGIC). */
+ uint32_t u32Magic;
+ /** the linux semaphore. */
+ struct semaphore Semaphore;
+#if defined(RT_STRICT) || defined(IPRT_DEBUG_SEMS)
+ /** For check. */
+ RTNATIVETHREAD volatile Owner;
+#endif
+} RTSEMFASTMUTEXINTERNAL, *PRTSEMFASTMUTEXINTERNAL;
+
+
+RTDECL(int) RTSemFastMutexCreate(PRTSEMFASTMUTEX phFastMtx)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Allocate.
+ */
+ PRTSEMFASTMUTEXINTERNAL pThis;
+ pThis = (PRTSEMFASTMUTEXINTERNAL)RTMemAlloc(sizeof(*pThis));
+ if (!pThis)
+ return VERR_NO_MEMORY;
+
+ /*
+ * Initialize.
+ */
+ pThis->u32Magic = RTSEMFASTMUTEX_MAGIC;
+ sema_init(&pThis->Semaphore, 1);
+#if defined(RT_STRICT) || defined(IPRT_DEBUG_SEMS)
+ pThis->Owner = NIL_RTNATIVETHREAD;
+#endif
+
+ *phFastMtx = pThis;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemFastMutexCreate);
+
+
+RTDECL(int) RTSemFastMutexDestroy(RTSEMFASTMUTEX hFastMtx)
+{
+ /*
+ * Validate.
+ */
+ PRTSEMFASTMUTEXINTERNAL pThis = hFastMtx;
+ if (pThis == NIL_RTSEMFASTMUTEX)
+ return VINF_SUCCESS;
+ AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+ AssertMsgReturn(pThis->u32Magic == RTSEMFASTMUTEX_MAGIC, ("u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+
+ ASMAtomicWriteU32(&pThis->u32Magic, RTSEMFASTMUTEX_MAGIC_DEAD);
+ RTMemFree(pThis);
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemFastMutexDestroy);
+
+
+RTDECL(int) RTSemFastMutexRequest(RTSEMFASTMUTEX hFastMtx)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate.
+ */
+ PRTSEMFASTMUTEXINTERNAL pThis = hFastMtx;
+ AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+ AssertMsgReturn(pThis->u32Magic == RTSEMFASTMUTEX_MAGIC, ("u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+
+ IPRT_DEBUG_SEMS_STATE(pThis, 'd');
+ down(&pThis->Semaphore);
+#if defined(RT_STRICT) || defined(IPRT_DEBUG_SEMS)
+ IPRT_DEBUG_SEMS_STATE(pThis, 'o');
+ AssertRelease(pThis->Owner == NIL_RTNATIVETHREAD);
+ ASMAtomicUoWriteSize(&pThis->Owner, RTThreadNativeSelf());
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemFastMutexRequest);
+
+
+RTDECL(int) RTSemFastMutexRelease(RTSEMFASTMUTEX hFastMtx)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate.
+ */
+ PRTSEMFASTMUTEXINTERNAL pThis = hFastMtx;
+ AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+ AssertMsgReturn(pThis->u32Magic == RTSEMFASTMUTEX_MAGIC, ("u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+
+#if defined(RT_STRICT) || defined(IPRT_DEBUG_SEMS)
+ AssertRelease(pThis->Owner == RTThreadNativeSelf());
+ ASMAtomicUoWriteSize(&pThis->Owner, NIL_RTNATIVETHREAD);
+#endif
+ up(&pThis->Semaphore);
+ IPRT_DEBUG_SEMS_STATE(pThis, 'u');
+
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemFastMutexRelease);
+
diff --git a/src/VBox/Runtime/r0drv/linux/semmutex-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/semmutex-r0drv-linux.c
new file mode 100644
index 00000000..16fcacfe
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/semmutex-r0drv-linux.c
@@ -0,0 +1,431 @@
+/* $Id: semmutex-r0drv-linux.c $ */
+/** @file
+ * IPRT - Mutex Semaphores, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#define RTSEMMUTEX_WITHOUT_REMAPPING
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/semaphore.h>
+
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/mem.h>
+#include <iprt/err.h>
+#include <iprt/list.h>
+
+#include "internal/magics.h"
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+typedef struct RTSEMMUTEXLNXWAITER
+{
+ /** The list entry. */
+ RTLISTNODE ListEntry;
+ /** The waiting task. */
+ struct task_struct *pTask;
+ /** Why did we wake up? */
+ enum
+ {
+ /** Wakeup to take the semaphore. */
+ RTSEMMUTEXLNXWAITER_WAKEUP,
+ /** Mutex is being destroyed. */
+ RTSEMMUTEXLNXWAITER_DESTROYED,
+ /** Some other reason. */
+ RTSEMMUTEXLNXWAITER_OTHER
+ } volatile enmReason;
+} RTSEMMUTEXLNXWAITER, *PRTSEMMUTEXLNXWAITER;
+
+/**
+ * Wrapper for the linux semaphore structure.
+ */
+typedef struct RTSEMMUTEXINTERNAL
+{
+ /** Magic value (RTSEMMUTEX_MAGIC). */
+ uint32_t u32Magic;
+ /** The number of recursions. */
+ uint32_t cRecursions;
+ /** The list of waiting threads. */
+ RTLISTANCHOR WaiterList;
+ /** The current owner, NULL if none. */
+ struct task_struct *pOwnerTask;
+ /** The number of references to this piece of memory. This is used to
+ * prevent it from being kicked from underneath us while waiting. */
+ uint32_t volatile cRefs;
+ /** The spinlock protecting the members and falling asleep. */
+ spinlock_t Spinlock;
+} RTSEMMUTEXINTERNAL, *PRTSEMMUTEXINTERNAL;
+
+
+RTDECL(int) RTSemMutexCreate(PRTSEMMUTEX phMtx)
+{
+ int rc = VINF_SUCCESS;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Allocate.
+ */
+ PRTSEMMUTEXINTERNAL pThis;
+ pThis = (PRTSEMMUTEXINTERNAL)RTMemAlloc(sizeof(*pThis));
+ if (pThis)
+ {
+ /*
+ * Initialize.
+ */
+ pThis->u32Magic = RTSEMMUTEX_MAGIC;
+ pThis->cRecursions = 0;
+ pThis->pOwnerTask = NULL;
+ pThis->cRefs = 1;
+ RTListInit(&pThis->WaiterList);
+ spin_lock_init(&pThis->Spinlock);
+
+ *phMtx = pThis;
+ }
+ else
+ rc = VERR_NO_MEMORY;
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+RT_EXPORT_SYMBOL(RTSemMutexCreate);
+
+
+RTDECL(int) RTSemMutexDestroy(RTSEMMUTEX hMtx)
+{
+ PRTSEMMUTEXINTERNAL pThis = hMtx;
+ PRTSEMMUTEXLNXWAITER pCur;
+ unsigned long fSavedIrq;
+
+ /*
+ * Validate.
+ */
+ if (pThis == NIL_RTSEMMUTEX)
+ return VINF_SUCCESS;
+ AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+ AssertMsgReturn(pThis->u32Magic == RTSEMMUTEX_MAGIC, ("u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+
+ /*
+ * Kill it, kick waiters and release it.
+ */
+ AssertReturn(ASMAtomicCmpXchgU32(&pThis->u32Magic, RTSEMMUTEX_MAGIC_DEAD, RTSEMMUTEX_MAGIC), VERR_INVALID_HANDLE);
+
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ spin_lock_irqsave(&pThis->Spinlock, fSavedIrq);
+ RTListForEach(&pThis->WaiterList, pCur, RTSEMMUTEXLNXWAITER, ListEntry)
+ {
+ pCur->enmReason = RTSEMMUTEXLNXWAITER_DESTROYED;
+ wake_up_process(pCur->pTask);
+ }
+
+ if (ASMAtomicDecU32(&pThis->cRefs) != 0)
+ spin_unlock_irqrestore(&pThis->Spinlock, fSavedIrq);
+ else
+ {
+ spin_unlock_irqrestore(&pThis->Spinlock, fSavedIrq);
+ RTMemFree(pThis);
+ }
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSemMutexDestroy);
+
+
+/**
+ * Worker for rtSemMutexLinuxRequest that handles the case where we go to sleep.
+ *
+ * @returns VINF_SUCCESS, VERR_INTERRUPTED, VERR_TIMEOUT or VERR_SEM_DESTROYED.
+ * Returns without owning the spinlock.
+ * @param pThis The mutex instance.
+ * @param cMillies The timeout.
+ * @param fInterruptible The wait type.
+ * @param fSavedIrq The saved IRQ flags.
+ */
+static int rtSemMutexLinuxRequestSleep(PRTSEMMUTEXINTERNAL pThis, RTMSINTERVAL cMillies,
+ bool fInterruptible, unsigned long fSavedIrq)
+{
+ struct task_struct *pSelf = current;
+ int rc = VERR_TIMEOUT;
+ long lTimeout = cMillies == RT_INDEFINITE_WAIT ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(cMillies);
+ RTSEMMUTEXLNXWAITER Waiter;
+
+ IPRT_DEBUG_SEMS_STATE(pThis, 'm');
+
+ /*
+ * Grab a reference to the mutex and add ourselves to the waiter list.
+ */
+ ASMAtomicIncU32(&pThis->cRefs);
+
+ Waiter.pTask = pSelf;
+ Waiter.enmReason = RTSEMMUTEXLNXWAITER_OTHER;
+ RTListAppend(&pThis->WaiterList, &Waiter.ListEntry);
+
+ /*
+ * Do the waiting.
+ */
+ for (;;)
+ {
+ /* Check signal and timeout conditions. */
+ if ( fInterruptible
+ && signal_pending(pSelf))
+ {
+ rc = VERR_INTERRUPTED;
+ break;
+ }
+
+ if (!lTimeout)
+ break;
+
+ /* Go to sleep. */
+ set_current_state(fInterruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
+ spin_unlock_irq(&pThis->Spinlock);
+
+ lTimeout = schedule_timeout(lTimeout);
+
+ spin_lock_irq(&pThis->Spinlock);
+ set_current_state(TASK_RUNNING);
+
+ /* Did someone wake us up? */
+ if (Waiter.enmReason == RTSEMMUTEXLNXWAITER_WAKEUP)
+ {
+ Assert(pThis->cRecursions == 0);
+ pThis->cRecursions = 1;
+ pThis->pOwnerTask = pSelf;
+ rc = VINF_SUCCESS;
+ break;
+ }
+
+ /* Is the mutex being destroyed? */
+ if (RT_UNLIKELY( Waiter.enmReason == RTSEMMUTEXLNXWAITER_DESTROYED
+ || pThis->u32Magic != RTSEMMUTEX_MAGIC))
+ {
+ rc = VERR_SEM_DESTROYED;
+ break;
+ }
+ }
+
+ /*
+ * Unlink ourself from the waiter list, dereference the mutex and exit the
+ * lock. We might have to free the mutex if it was the destroyed.
+ */
+ RTListNodeRemove(&Waiter.ListEntry);
+ IPRT_DEBUG_SEMS_STATE_RC(pThis, 'M', rc);
+
+ if (RT_LIKELY(ASMAtomicDecU32(&pThis->cRefs) != 0))
+ spin_unlock_irqrestore(&pThis->Spinlock, fSavedIrq);
+ else
+ {
+ Assert(RT_FAILURE_NP(rc));
+ spin_unlock_irqrestore(&pThis->Spinlock, fSavedIrq);
+ RTMemFree(pThis);
+ }
+ return rc;
+}
+
+
+/**
+ * Internal worker.
+ */
+DECLINLINE(int) rtSemMutexLinuxRequest(RTSEMMUTEX hMutexSem, RTMSINTERVAL cMillies, bool fInterruptible)
+{
+ PRTSEMMUTEXINTERNAL pThis = hMutexSem;
+ struct task_struct *pSelf = current;
+ unsigned long fSavedIrq;
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate.
+ */
+ AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+ AssertMsgReturn(pThis->u32Magic == RTSEMMUTEX_MAGIC, ("u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+ Assert(pThis->cRefs >= 1);
+
+ /*
+ * Lock it and check if it's a recursion.
+ */
+ spin_lock_irqsave(&pThis->Spinlock, fSavedIrq);
+ if (pThis->pOwnerTask == pSelf)
+ {
+ pThis->cRecursions++;
+ Assert(pThis->cRecursions > 1);
+ Assert(pThis->cRecursions < 256);
+ rc = VINF_SUCCESS;
+ }
+ /*
+ * Not a recursion, maybe it's not owned by anyone then?
+ */
+ else if ( pThis->pOwnerTask == NULL
+ && RTListIsEmpty(&pThis->WaiterList))
+ {
+ Assert(pThis->cRecursions == 0);
+ pThis->cRecursions = 1;
+ pThis->pOwnerTask = pSelf;
+ rc = VINF_SUCCESS;
+ }
+ /*
+ * Was it a polling call?
+ */
+ else if (cMillies == 0)
+ rc = VERR_TIMEOUT;
+ /*
+ * No, so go to sleep.
+ */
+ else
+ {
+ rc = rtSemMutexLinuxRequestSleep(pThis, cMillies, fInterruptible, fSavedIrq);
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC();
+ return rc;
+ }
+
+ IPRT_DEBUG_SEMS_STATE_RC(pThis, 'M', rc);
+ spin_unlock_irqrestore(&pThis->Spinlock, fSavedIrq);
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC();
+ return rc;
+}
+
+
+RTDECL(int) RTSemMutexRequest(RTSEMMUTEX hMutexSem, RTMSINTERVAL cMillies)
+{
+ return rtSemMutexLinuxRequest(hMutexSem, cMillies, false /*fInterruptible*/);
+}
+RT_EXPORT_SYMBOL(RTSemMutexRequest);
+
+
+RTDECL(int) RTSemMutexRequestDebug(RTSEMMUTEX hMutexSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+ RT_NOREF_PV(uId); RT_SRC_POS_NOREF();
+ return RTSemMutexRequest(hMutexSem, cMillies);
+}
+RT_EXPORT_SYMBOL(RTSemMutexRequestDebug);
+
+
+RTDECL(int) RTSemMutexRequestNoResume(RTSEMMUTEX hMutexSem, RTMSINTERVAL cMillies)
+{
+ return rtSemMutexLinuxRequest(hMutexSem, cMillies, true /*fInterruptible*/);
+}
+RT_EXPORT_SYMBOL(RTSemMutexRequestNoResume);
+
+
+RTDECL(int) RTSemMutexRequestNoResumeDebug(RTSEMMUTEX hMutexSem, RTMSINTERVAL cMillies, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+ RT_NOREF_PV(uId); RT_SRC_POS_NOREF();
+ return RTSemMutexRequestNoResume(hMutexSem, cMillies);
+}
+RT_EXPORT_SYMBOL(RTSemMutexRequestNoResumeDebug);
+
+
+RTDECL(int) RTSemMutexRelease(RTSEMMUTEX hMtx)
+{
+ PRTSEMMUTEXINTERNAL pThis = hMtx;
+ struct task_struct *pSelf = current;
+ unsigned long fSavedIrq;
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate.
+ */
+ AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
+ AssertMsgReturn(pThis->u32Magic == RTSEMMUTEX_MAGIC, ("u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), VERR_INVALID_HANDLE);
+ Assert(pThis->cRefs >= 1);
+
+ /*
+ * Take the lock and release one recursion.
+ */
+ spin_lock_irqsave(&pThis->Spinlock, fSavedIrq);
+ if (pThis->pOwnerTask == pSelf)
+ {
+ Assert(pThis->cRecursions > 0);
+ if (--pThis->cRecursions == 0)
+ {
+ pThis->pOwnerTask = NULL;
+
+ /* anyone to wake up? */
+ if (!RTListIsEmpty(&pThis->WaiterList))
+ {
+ PRTSEMMUTEXLNXWAITER pWaiter = RTListGetFirst(&pThis->WaiterList, RTSEMMUTEXLNXWAITER, ListEntry);
+ pWaiter->enmReason = RTSEMMUTEXLNXWAITER_WAKEUP;
+ wake_up_process(pWaiter->pTask);
+ }
+ IPRT_DEBUG_SEMS_STATE(pThis, 'u');
+ }
+ rc = VINF_SUCCESS;
+ }
+ else
+ rc = VERR_NOT_OWNER;
+ spin_unlock_irqrestore(&pThis->Spinlock, fSavedIrq);
+
+ AssertRC(rc);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+}
+RT_EXPORT_SYMBOL(RTSemMutexRelease);
+
+
+RTDECL(bool) RTSemMutexIsOwned(RTSEMMUTEX hMutexSem)
+{
+ PRTSEMMUTEXINTERNAL pThis = hMutexSem;
+ unsigned long fSavedIrq;
+ bool fOwned;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate.
+ */
+ AssertPtrReturn(pThis, false);
+ AssertMsgReturn(pThis->u32Magic == RTSEMMUTEX_MAGIC, ("u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis), false);
+ Assert(pThis->cRefs >= 1);
+
+ /*
+ * Take the lock and release one recursion.
+ */
+ spin_lock_irqsave(&pThis->Spinlock, fSavedIrq);
+ fOwned = pThis->pOwnerTask != NULL;
+ spin_unlock_irqrestore(&pThis->Spinlock, fSavedIrq);
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return fOwned;
+
+}
+RT_EXPORT_SYMBOL(RTSemMutexIsOwned);
+
diff --git a/src/VBox/Runtime/r0drv/linux/spinlock-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/spinlock-r0drv-linux.c
new file mode 100644
index 00000000..c5bdded2
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/spinlock-r0drv-linux.c
@@ -0,0 +1,196 @@
+/* $Id: spinlock-r0drv-linux.c $ */
+/** @file
+ * IPRT - Spinlocks, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/spinlock.h>
+
+#include <iprt/asm.h>
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h>
+#endif
+#include <iprt/assert.h>
+#include <iprt/errcore.h>
+#include <iprt/mem.h>
+#include <iprt/mp.h>
+#include <iprt/thread.h>
+#include "internal/magics.h"
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+/**
+ * Wrapper for the spinlock_t structure.
+ */
+typedef struct RTSPINLOCKINTERNAL
+{
+ /** Spinlock magic value (RTSPINLOCK_MAGIC). */
+ uint32_t volatile u32Magic;
+ /** The spinlock creation flags. */
+ uint32_t fFlags;
+ /** The saved interrupt flag. */
+ unsigned long volatile fIntSaved;
+ /** The linux spinlock structure. */
+ spinlock_t Spinlock;
+#ifdef RT_MORE_STRICT
+ /** The idAssertCpu variable before acquring the lock for asserting after
+ * releasing the spinlock. */
+ RTCPUID volatile idAssertCpu;
+ /** The CPU that owns the lock. */
+ RTCPUID volatile idCpuOwner;
+#endif
+} RTSPINLOCKINTERNAL, *PRTSPINLOCKINTERNAL;
+
+
+
+RTDECL(int) RTSpinlockCreate(PRTSPINLOCK pSpinlock, uint32_t fFlags, const char *pszName)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ PRTSPINLOCKINTERNAL pThis;
+ AssertReturn(fFlags == RTSPINLOCK_FLAGS_INTERRUPT_SAFE || fFlags == RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, VERR_INVALID_PARAMETER);
+ RT_NOREF_PV(pszName);
+
+ /*
+ * Allocate.
+ */
+ Assert(sizeof(RTSPINLOCKINTERNAL) > sizeof(void *));
+ pThis = (PRTSPINLOCKINTERNAL)RTMemAlloc(sizeof(*pThis));
+ if (!pThis)
+ return VERR_NO_MEMORY;
+ /*
+ * Initialize and return.
+ */
+ pThis->u32Magic = RTSPINLOCK_MAGIC;
+ pThis->fFlags = fFlags;
+ pThis->fIntSaved = 0;
+#ifdef RT_MORE_STRICT
+ pThis->idCpuOwner = NIL_RTCPUID;
+ pThis->idAssertCpu = NIL_RTCPUID;
+#endif
+
+ spin_lock_init(&pThis->Spinlock);
+
+ *pSpinlock = pThis;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSpinlockCreate);
+
+
+RTDECL(int) RTSpinlockDestroy(RTSPINLOCK Spinlock)
+{
+ /*
+ * Validate input.
+ */
+ PRTSPINLOCKINTERNAL pThis = (PRTSPINLOCKINTERNAL)Spinlock;
+ if (!pThis)
+ return VERR_INVALID_PARAMETER;
+ if (pThis->u32Magic != RTSPINLOCK_MAGIC)
+ {
+ AssertMsgFailed(("Invalid spinlock %p magic=%#x\n", pThis, pThis->u32Magic));
+ return VERR_INVALID_PARAMETER;
+ }
+
+ ASMAtomicIncU32(&pThis->u32Magic);
+ RTMemFree(pThis);
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTSpinlockDestroy);
+
+
+RTDECL(void) RTSpinlockAcquire(RTSPINLOCK Spinlock)
+{
+ PRTSPINLOCKINTERNAL pThis = (PRTSPINLOCKINTERNAL)Spinlock;
+ IPRT_LINUX_SAVE_EFL_AC();
+ RT_ASSERT_PREEMPT_CPUID_VAR();
+ AssertMsg(pThis && pThis->u32Magic == RTSPINLOCK_MAGIC,
+ ("pThis=%p u32Magic=%08x\n", pThis, pThis ? (int)pThis->u32Magic : 0));
+
+#ifdef CONFIG_PROVE_LOCKING
+ lockdep_off();
+#endif
+ if (pThis->fFlags & RTSPINLOCK_FLAGS_INTERRUPT_SAFE)
+ {
+ unsigned long fIntSaved;
+ spin_lock_irqsave(&pThis->Spinlock, fIntSaved);
+ pThis->fIntSaved = fIntSaved;
+ }
+ else
+ spin_lock(&pThis->Spinlock);
+#ifdef CONFIG_PROVE_LOCKING
+ lockdep_on();
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC();
+ RT_ASSERT_PREEMPT_CPUID_SPIN_ACQUIRED(pThis);
+}
+RT_EXPORT_SYMBOL(RTSpinlockAcquire);
+
+
+RTDECL(void) RTSpinlockRelease(RTSPINLOCK Spinlock)
+{
+ PRTSPINLOCKINTERNAL pThis = (PRTSPINLOCKINTERNAL)Spinlock;
+ IPRT_LINUX_SAVE_EFL_AC(); /* spin_unlock* may preempt and trash eflags.ac. */
+ RT_ASSERT_PREEMPT_CPUID_SPIN_RELEASE_VARS();
+ AssertMsg(pThis && pThis->u32Magic == RTSPINLOCK_MAGIC,
+ ("pThis=%p u32Magic=%08x\n", pThis, pThis ? (int)pThis->u32Magic : 0));
+ RT_ASSERT_PREEMPT_CPUID_SPIN_RELEASE(pThis);
+
+#ifdef CONFIG_PROVE_LOCKING
+ lockdep_off();
+#endif
+ if (pThis->fFlags & RTSPINLOCK_FLAGS_INTERRUPT_SAFE)
+ {
+ unsigned long fIntSaved = pThis->fIntSaved;
+ pThis->fIntSaved = 0;
+ spin_unlock_irqrestore(&pThis->Spinlock, fIntSaved);
+ }
+ else
+ spin_unlock(&pThis->Spinlock);
+#ifdef CONFIG_PROVE_LOCKING
+ lockdep_on();
+#endif
+
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC();
+ RT_ASSERT_PREEMPT_CPUID();
+}
+RT_EXPORT_SYMBOL(RTSpinlockRelease);
+
diff --git a/src/VBox/Runtime/r0drv/linux/string.h b/src/VBox/Runtime/r0drv/linux/string.h
new file mode 100644
index 00000000..9e5a6941
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/string.h
@@ -0,0 +1,70 @@
+/* $Id: string.h $ */
+/** @file
+ * IPRT - wrapper for the linux kernel asm/string.h.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+#ifndef IPRT_INCLUDED_SRC_r0drv_linux_string_h
+#define IPRT_INCLUDED_SRC_r0drv_linux_string_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include <iprt/cdefs.h>
+
+RT_C_DECLS_BEGIN
+#ifndef bool /* Linux 2.6.19 C++ nightmare */
+#define bool bool_type
+#define true true_type
+#define false false_type
+#define _Bool int
+#define bool_type_r0drv_string_h__
+#endif
+#include <linux/types.h>
+#include <linux/string.h>
+#ifdef bool_type_r0drv_string_h__
+#undef bool
+#undef true
+#undef false
+#undef bool_type_r0drv_string_h__
+#endif
+char *strpbrk(const char *pszStr, const char *pszChars)
+#if defined(__THROW)
+ __THROW
+#endif
+ ;
+
+RT_C_DECLS_END
+
+#endif /* !IPRT_INCLUDED_SRC_r0drv_linux_string_h */
+
diff --git a/src/VBox/Runtime/r0drv/linux/the-linux-kernel.h b/src/VBox/Runtime/r0drv/linux/the-linux-kernel.h
new file mode 100644
index 00000000..8e70f992
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/the-linux-kernel.h
@@ -0,0 +1,494 @@
+/* $Id: the-linux-kernel.h $ */
+/** @file
+ * IPRT - Include all necessary headers for the Linux kernel.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+#ifndef IPRT_INCLUDED_SRC_r0drv_linux_the_linux_kernel_h
+#define IPRT_INCLUDED_SRC_r0drv_linux_the_linux_kernel_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+/*
+ * Include iprt/types.h to install the bool wrappers.
+ * Then use the linux bool type for all the stuff include here.
+ */
+#include <iprt/types.h>
+#define bool linux_bool
+
+#if RT_GNUC_PREREQ(4, 6)
+# pragma GCC diagnostic push
+#endif
+#if RT_GNUC_PREREQ(4, 2)
+# pragma GCC diagnostic ignored "-Wunused-parameter"
+# if !defined(__cplusplus) && RT_GNUC_PREREQ(4, 3)
+# pragma GCC diagnostic ignored "-Wold-style-declaration" /* 2.6.18-411.0.0.0.1.el5/build/include/asm/apic.h:110: warning: 'inline' is not at beginning of declaration [-Wold-style-declaration] */
+# endif
+#endif
+
+
+#include <iprt/linux/version.h>
+#if RTLNX_VER_MIN(2,6,33)
+# include <generated/autoconf.h>
+#else
+# ifndef AUTOCONF_INCLUDED
+# include <linux/autoconf.h>
+# endif
+#endif
+
+/* We only support 2.4 and 2.6 series kernels */
+#if RTLNX_VER_MAX(2,4,0)
+# error Sorry, we do not support 2.3 and earlier kernels.
+#endif
+#if RTLNX_VER_MIN(2,5,0) && RTLNX_VER_MAX(2,6,0)
+# error Sorry, we do not support 2.5 series kernels (might work though).
+#endif
+
+#if defined(CONFIG_MODVERSIONS) && !defined(MODVERSIONS)
+# define MODVERSIONS
+# if RTLNX_VER_MAX(2,5,71)
+# include <linux/modversions.h>
+# endif
+#endif
+#ifndef KBUILD_STR
+# if RTLNX_VER_MAX(2,6,16)
+# define KBUILD_STR(s) s
+# else
+# define KBUILD_STR(s) #s
+# endif
+#endif
+# if RTLNX_VER_MIN(3,3,0)
+# include <linux/kconfig.h> /* for macro IS_ENABLED */
+# endif
+#include <linux/string.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#if RTLNX_VER_MIN(2,6,27)
+# include <linux/semaphore.h>
+#else /* older kernels */
+# include <asm/semaphore.h>
+#endif /* older kernels */
+#include <linux/module.h>
+#if RTLNX_VER_MIN(2,6,0)
+# include <linux/moduleparam.h>
+#endif
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#if RTLNX_VER_MIN(2,6,0)
+# include <linux/namei.h>
+#endif
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/time.h>
+#include <linux/sched.h>
+
+#if RTLNX_VER_RANGE(3,9,23, 3,9,31)
+# include <linux/splice.h>
+#endif
+
+#if RTLNX_VER_MIN(3,9,0)
+# include <linux/sched/rt.h>
+#endif
+#if RTLNX_VER_MIN(4,11,0)
+# include <linux/sched/signal.h>
+# include <linux/sched/types.h>
+#endif
+#if RTLNX_VER_MIN(2,6,7)
+# include <linux/jiffies.h>
+#endif
+#if RTLNX_VER_MIN(2,6,16)
+# include <linux/ktime.h>
+# include <linux/hrtimer.h>
+#endif
+#include <linux/wait.h>
+#if RTLNX_VER_MIN(2,5,71)
+# include <linux/cpu.h>
+# include <linux/notifier.h>
+#endif
+#if RTLNX_VER_MIN(5,1,0)
+# include <uapi/linux/mman.h>
+#endif
+/* For the basic additions module */
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/completion.h>
+#include <linux/compiler.h>
+#if RTLNX_VER_MIN(5,9,0) || RTLNX_SUSE_MAJ_PREREQ(15,3) /* linux/fs.h defined HAVE_UNLOCKED_IOCTL from 2.6.11 up to 5.9 (also 5.3.18-56 in SLES15-SP3), when it became an implicit assumption. */
+# define HAVE_UNLOCKED_IOCTL 1 /* We use this in a couple of places, so for now just define it for 5.9+ too. */
+#endif
+#if !defined(HAVE_UNLOCKED_IOCTL) && RTLNX_VER_MAX(2,6,38)
+# include <linux/smp_lock.h>
+#endif
+/* For the shared folders module */
+#include <linux/vmalloc.h>
+#define wchar_t linux_wchar_t
+#include <linux/nls.h>
+#undef wchar_t
+#include <asm/mman.h>
+#include <asm/io.h>
+#include <asm/uaccess.h>
+#include <asm/div64.h>
+
+/* For thread-context hooks. */
+#if RTLNX_VER_MIN(2,6,18) && defined(CONFIG_PREEMPT_NOTIFIERS)
+# include <linux/preempt.h>
+#endif
+
+/* for workqueue / task queues. */
+#if RTLNX_VER_MIN(2,5,41)
+# include <linux/workqueue.h>
+#else
+# include <linux/tqueue.h>
+#endif
+
+#if RTLNX_VER_MIN(2,6,4)
+# include <linux/kthread.h>
+#endif
+
+/* for cr4_init_shadow() / cpu_tlbstate. */
+#if RTLNX_VER_MIN(3,20,0)
+# include <asm/tlbflush.h>
+#endif
+
+/* for set_pages_x() */
+#if RTLNX_VER_MIN(4,12,0)
+# include <asm/set_memory.h>
+#endif
+
+/* for __flush_tlb_all() */
+#if RTLNX_VER_MIN(2,6,28) && (defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86))
+# include <asm/tlbflush.h>
+#endif
+
+/* for kernel_fpu_begin / kernel_fpu_end() */
+#if RTLNX_VER_MIN(4,2,0)
+# include <asm/fpu/api.h>
+#endif
+
+#if RTLNX_VER_MIN(3,7,0)
+# include <asm/smap.h>
+#else
+static inline void clac(void) { }
+static inline void stac(void) { }
+#endif
+
+#if RTLNX_VER_MAX(2,6,0)
+# ifndef page_to_pfn
+# define page_to_pfn(page) ((page) - mem_map)
+# endif
+#endif
+
+#ifndef DEFINE_WAIT
+# define DEFINE_WAIT(name) DECLARE_WAITQUEUE(name, current)
+#endif
+
+#ifndef __GFP_NOWARN
+# define __GFP_NOWARN 0
+#endif
+
+/*
+ * 2.4 / early 2.6 compatibility wrappers
+ */
+#if RTLNX_VER_MAX(2,6,7)
+
+# ifndef MAX_JIFFY_OFFSET
+# define MAX_JIFFY_OFFSET ((~0UL >> 1)-1)
+# endif
+
+# if RTLNX_VER_MAX(2,4,29) || RTLNX_VER_MIN(2,6,0)
+
+DECLINLINE(unsigned int) jiffies_to_msecs(unsigned long cJiffies)
+{
+# if HZ <= 1000 && !(1000 % HZ)
+ return (1000 / HZ) * cJiffies;
+# elif HZ > 1000 && !(HZ % 1000)
+ return (cJiffies + (HZ / 1000) - 1) / (HZ / 1000);
+# else
+ return (cJiffies * 1000) / HZ;
+# endif
+}
+
+DECLINLINE(unsigned long) msecs_to_jiffies(unsigned int cMillies)
+{
+# if HZ > 1000
+ if (cMillies > jiffies_to_msecs(MAX_JIFFY_OFFSET))
+ return MAX_JIFFY_OFFSET;
+# endif
+# if HZ <= 1000 && !(1000 % HZ)
+ return (cMillies + (1000 / HZ) - 1) / (1000 / HZ);
+# elif HZ > 1000 && !(HZ % 1000)
+ return cMillies * (HZ / 1000);
+# else
+ return (cMillies * HZ + 999) / 1000;
+# endif
+}
+
+# endif /* < 2.4.29 || >= 2.6.0 */
+
+#endif /* < 2.6.7 */
+
+/*
+ * 2.4 compatibility wrappers
+ */
+#if RTLNX_VER_MAX(2,6,0)
+
+# define prepare_to_wait(q, wait, state) \
+ do { \
+ add_wait_queue(q, wait); \
+ set_current_state(state); \
+ } while (0)
+
+# define after_wait(wait) \
+ do { \
+ list_del_init(&(wait)->task_list); \
+ } while (0)
+
+# define finish_wait(q, wait) \
+ do { \
+ set_current_state(TASK_RUNNING); \
+ remove_wait_queue(q, wait); \
+ } while (0)
+
+#else /* >= 2.6.0 */
+
+# define after_wait(wait) do {} while (0)
+
+#endif /* >= 2.6.0 */
+
+/** @def TICK_NSEC
+ * The time between ticks in nsec */
+#ifndef TICK_NSEC
+# define TICK_NSEC (1000000000UL / HZ)
+#endif
+
+/*
+ * This sucks soooo badly on x86! Why don't they export __PAGE_KERNEL_EXEC so PAGE_KERNEL_EXEC would be usable?
+ */
+#if RTLNX_VER_MIN(2,6,8) && defined(RT_ARCH_AMD64)
+# define MY_PAGE_KERNEL_EXEC PAGE_KERNEL_EXEC
+#elif RTLNX_VER_MIN(2,6,8) && defined(PAGE_KERNEL_EXEC) && defined(CONFIG_X86_PAE)
+# ifdef __PAGE_KERNEL_EXEC
+ /* >= 2.6.27 */
+# define MY_PAGE_KERNEL_EXEC __pgprot(boot_cpu_has(X86_FEATURE_PGE) ? __PAGE_KERNEL_EXEC | _PAGE_GLOBAL : __PAGE_KERNEL_EXEC)
+# else
+# define MY_PAGE_KERNEL_EXEC __pgprot(boot_cpu_has(X86_FEATURE_PGE) ? _PAGE_KERNEL_EXEC | _PAGE_GLOBAL : _PAGE_KERNEL_EXEC)
+# endif
+#else
+# define MY_PAGE_KERNEL_EXEC PAGE_KERNEL
+#endif
+
+
+/*
+ * The redhat hack section.
+ * - The current hacks are for 2.4.21-15.EL only.
+ */
+#ifndef NO_REDHAT_HACKS
+/* accounting. */
+# if RTLNX_VER_MAX(2,6,0)
+# ifdef VM_ACCOUNT
+# define USE_RHEL4_MUNMAP
+# endif
+# endif
+
+/* backported remap_page_range. */
+# if RTLNX_VER_MAX(2,6,0)
+# include <asm/tlb.h>
+# ifdef tlb_vma /* probably not good enough... */
+# define HAVE_26_STYLE_REMAP_PAGE_RANGE 1
+# endif
+# endif
+
+# ifndef RT_ARCH_AMD64
+/* In 2.6.9-22.ELsmp we have to call change_page_attr() twice when changing
+ * the page attributes from PAGE_KERNEL to something else, because there appears
+ * to be a bug in one of the many patches that redhat applied.
+ * It should be safe to do this on less buggy linux kernels too. ;-)
+ */
+# define MY_CHANGE_PAGE_ATTR(pPages, cPages, prot) \
+ do { \
+ if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) \
+ change_page_attr(pPages, cPages, prot); \
+ change_page_attr(pPages, cPages, prot); \
+ } while (0)
+# endif /* !RT_ARCH_AMD64 */
+#endif /* !NO_REDHAT_HACKS */
+
+#ifndef MY_CHANGE_PAGE_ATTR
+# ifdef RT_ARCH_AMD64 /** @todo This is a cheap hack, but it'll get around that 'else BUG();' in __change_page_attr(). */
+# define MY_CHANGE_PAGE_ATTR(pPages, cPages, prot) \
+ do { \
+ change_page_attr(pPages, cPages, PAGE_KERNEL_NOCACHE); \
+ change_page_attr(pPages, cPages, prot); \
+ } while (0)
+# else
+# define MY_CHANGE_PAGE_ATTR(pPages, cPages, prot) change_page_attr(pPages, cPages, prot)
+# endif
+#endif
+
+#if RTLNX_VER_MIN(2,6,25)
+# if RTLNX_VER_MAX(5,4,0) /* The interface was removed, but we only need it for < 2.4.22, so who cares. */
+# define MY_SET_PAGES_EXEC(pPages, cPages) set_pages_x(pPages, cPages)
+# define MY_SET_PAGES_NOEXEC(pPages, cPages) set_pages_nx(pPages, cPages)
+# endif
+#else
+# define MY_SET_PAGES_EXEC(pPages, cPages) \
+ do { \
+ if (pgprot_val(MY_PAGE_KERNEL_EXEC) != pgprot_val(PAGE_KERNEL)) \
+ MY_CHANGE_PAGE_ATTR(pPages, cPages, MY_PAGE_KERNEL_EXEC); \
+ } while (0)
+# define MY_SET_PAGES_NOEXEC(pPages, cPages) \
+ do { \
+ if (pgprot_val(MY_PAGE_KERNEL_EXEC) != pgprot_val(PAGE_KERNEL)) \
+ MY_CHANGE_PAGE_ATTR(pPages, cPages, PAGE_KERNEL); \
+ } while (0)
+#endif
+
+/** @def ONE_MSEC_IN_JIFFIES
+ * The number of jiffies that make up 1 millisecond. Must be at least 1! */
+#if HZ <= 1000
+# define ONE_MSEC_IN_JIFFIES 1
+#elif !(HZ % 1000)
+# define ONE_MSEC_IN_JIFFIES (HZ / 1000)
+#else
+# define ONE_MSEC_IN_JIFFIES ((HZ + 999) / 1000)
+# error "HZ is not a multiple of 1000, the GIP stuff won't work right!"
+#endif
+
+/*
+ * Stop using the linux bool type.
+ */
+#undef bool
+
+#if RT_GNUC_PREREQ(4, 6)
+# pragma GCC diagnostic pop
+#endif
+
+/*
+ * There are post-2.6.24 kernels (confusingly with unchanged version number)
+ * which eliminate macros which were marked as deprecated.
+ */
+#ifndef __attribute_used__
+#define __attribute_used__ __used
+#endif
+
+/**
+ * Hack for shortening pointers on linux so we can stuff more stuff into the
+ * task_struct::comm field. This is used by the semaphore code but put here
+ * because we don't have any better place atm. Don't use outside IPRT, please.
+ */
+#ifdef RT_ARCH_AMD64
+# define IPRT_DEBUG_SEMS_ADDRESS(addr) ( ((long)(addr) & (long)~UINT64_C(0xfffffff000000000)) )
+#else
+# define IPRT_DEBUG_SEMS_ADDRESS(addr) ( (long)(addr) )
+#endif
+
+/**
+ * Puts semaphore info into the task_struct::comm field if IPRT_DEBUG_SEMS is
+ * defined.
+ */
+#ifdef IPRT_DEBUG_SEMS
+# define IPRT_DEBUG_SEMS_STATE(pThis, chState) \
+ snprintf(current->comm, sizeof(current->comm), "%c%lx", (chState), IPRT_DEBUG_SEMS_ADDRESS(pThis));
+#else
+# define IPRT_DEBUG_SEMS_STATE(pThis, chState) do { } while (0)
+#endif
+
+/**
+ * Puts semaphore info into the task_struct::comm field if IPRT_DEBUG_SEMS is
+ * defined.
+ */
+#ifdef IPRT_DEBUG_SEMS
+# define IPRT_DEBUG_SEMS_STATE_RC(pThis, chState, rc) \
+ snprintf(current->comm, sizeof(current->comm), "%c%lx:%d", (chState), IPRT_DEBUG_SEMS_ADDRESS(pThis), rc);
+#else
+# define IPRT_DEBUG_SEMS_STATE_RC(pThis, chState, rc) do { } while (0)
+#endif
+
+/** @name Macros for preserving EFLAGS.AC on 3.19+/amd64 paranoid.
+ * The AMD 64 switch_to in macro in arch/x86/include/asm/switch_to.h stopped
+ * restoring flags.
+ * @{ */
+#if (defined(CONFIG_X86_SMAP) || defined(RT_STRICT) || defined(IPRT_WITH_EFLAGS_AC_PRESERVING)) \
+ && !defined(IPRT_WITHOUT_EFLAGS_AC_PRESERVING)
+# include <iprt/asm-amd64-x86.h>
+# define IPRT_X86_EFL_AC RT_BIT(18)
+# define IPRT_LINUX_SAVE_EFL_AC() RTCCUINTREG fSavedEfl = ASMGetFlags()
+# define IPRT_LINUX_RESTORE_EFL_AC() ASMSetFlags(fSavedEfl)
+# define IPRT_LINUX_RESTORE_EFL_ONLY_AC() ASMChangeFlags(~IPRT_X86_EFL_AC, fSavedEfl & IPRT_X86_EFL_AC)
+#else
+# define IPRT_LINUX_SAVE_EFL_AC() do { } while (0)
+# define IPRT_LINUX_RESTORE_EFL_AC() do { } while (0)
+# define IPRT_LINUX_RESTORE_EFL_ONLY_AC() do { } while (0)
+#endif
+/** @} */
+
+/*
+ * There are some conflicting defines in iprt/param.h, sort them out here.
+ */
+#ifndef IPRT_INCLUDED_param_h
+# undef PAGE_SIZE
+# undef PAGE_OFFSET_MASK
+# include <iprt/param.h>
+#endif
+
+/*
+ * Some global indicator macros.
+ */
+/** @def IPRT_LINUX_HAS_HRTIMER
+ * Whether the kernel support high resolution timers (Linux kernel versions
+ * 2.6.28 and later (hrtimer_add_expires_ns() & schedule_hrtimeout). */
+#if RTLNX_VER_MIN(2,6,28) || defined(DOXYGEN_RUNNING)
+# define IPRT_LINUX_HAS_HRTIMER
+#endif
+
+/*
+ * Workqueue stuff, see initterm-r0drv-linux.c.
+ */
+#if RTLNX_VER_MIN(2,5,41)
+typedef struct work_struct RTR0LNXWORKQUEUEITEM;
+#else
+typedef struct tq_struct RTR0LNXWORKQUEUEITEM;
+#endif
+DECLHIDDEN(void) rtR0LnxWorkqueuePush(RTR0LNXWORKQUEUEITEM *pWork, void (*pfnWorker)(RTR0LNXWORKQUEUEITEM *));
+DECLHIDDEN(void) rtR0LnxWorkqueueFlush(void);
+
+/*
+ * Memory hacks from memobj-r0drv-linux.c that shared folders need.
+ */
+RTDECL(struct page *) rtR0MemObjLinuxVirtToPage(void *pv);
+
+#endif /* !IPRT_INCLUDED_SRC_r0drv_linux_the_linux_kernel_h */
diff --git a/src/VBox/Runtime/r0drv/linux/thread-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/thread-r0drv-linux.c
new file mode 100644
index 00000000..2724d091
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/thread-r0drv-linux.c
@@ -0,0 +1,285 @@
+/* $Id: thread-r0drv-linux.c $ */
+/** @file
+ * IPRT - Threads, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+#include <iprt/thread.h>
+
+#include <iprt/asm.h>
+#if RTLNX_VER_MAX(2,5,28) || defined(CONFIG_X86_SMAP)
+# include <iprt/asm-amd64-x86.h>
+#endif
+#include <iprt/assert.h>
+#include <iprt/errcore.h>
+#include <iprt/mp.h>
+
+
+/*********************************************************************************************************************************
+* Defined Constants And Macros *
+*********************************************************************************************************************************/
+#if defined(CONFIG_PREEMPT_COUNT) /* since 3.1 */ \
+ || defined(CONFIG_PREEMPTION) /* since 5.3 */ \
+ || defined(CONFIG_PREEMPT) /* since 2.6.13 - preemption model choice; before that arch specific choice back to 2.5.45. */
+# define IPRT_LNX_HAVE_PREEMPTION
+#endif
+
+
+/*********************************************************************************************************************************
+* Global Variables *
+*********************************************************************************************************************************/
+#ifndef IPRT_LNX_HAVE_PREEMPTION
+/** Per-cpu preemption counters. */
+static int32_t volatile g_acPreemptDisabled[NR_CPUS];
+#endif
+
+
+RTDECL(RTNATIVETHREAD) RTThreadNativeSelf(void)
+{
+ return (RTNATIVETHREAD)current;
+}
+RT_EXPORT_SYMBOL(RTThreadNativeSelf);
+
+
+static int rtR0ThreadLnxSleepCommon(RTMSINTERVAL cMillies)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+ long cJiffies = msecs_to_jiffies(cMillies);
+ set_current_state(TASK_INTERRUPTIBLE);
+ cJiffies = schedule_timeout(cJiffies);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ if (!cJiffies)
+ return VINF_SUCCESS;
+ return VERR_INTERRUPTED;
+}
+
+
+RTDECL(int) RTThreadSleep(RTMSINTERVAL cMillies)
+{
+ return rtR0ThreadLnxSleepCommon(cMillies);
+}
+RT_EXPORT_SYMBOL(RTThreadSleep);
+
+
+RTDECL(int) RTThreadSleepNoLog(RTMSINTERVAL cMillies)
+{
+ return rtR0ThreadLnxSleepCommon(cMillies);
+}
+RT_EXPORT_SYMBOL(RTThreadSleepNoLog);
+
+
+RTDECL(bool) RTThreadYield(void)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+#if RTLNX_VER_MIN(2,4,20)
+ yield();
+#else
+ /** @todo r=ramshankar: Can we use cond_resched() instead? */
+ set_current_state(TASK_RUNNING);
+ sys_sched_yield();
+ schedule();
+#endif
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return true;
+}
+RT_EXPORT_SYMBOL(RTThreadYield);
+
+
+RTDECL(bool) RTThreadPreemptIsEnabled(RTTHREAD hThread)
+{
+#ifdef IPRT_LNX_HAVE_PREEMPTION
+ Assert(hThread == NIL_RTTHREAD); RT_NOREF_PV(hThread);
+# ifdef preemptible
+ return preemptible();
+# else
+ return preempt_count() == 0 && !in_atomic() && !irqs_disabled();
+# endif
+#else
+ int32_t c;
+
+ Assert(hThread == NIL_RTTHREAD);
+ c = g_acPreemptDisabled[smp_processor_id()];
+ AssertMsg(c >= 0 && c < 32, ("%d\n", c));
+ if (c != 0)
+ return false;
+# if RTLNX_VER_MIN(2,5,32)
+ if (in_atomic())
+ return false;
+# endif
+# if RTLNX_VER_MIN(2,5,28)
+ if (irqs_disabled())
+ return false;
+# else
+ if (!ASMIntAreEnabled())
+ return false;
+# endif
+ return true;
+#endif
+}
+RT_EXPORT_SYMBOL(RTThreadPreemptIsEnabled);
+
+
+RTDECL(bool) RTThreadPreemptIsPending(RTTHREAD hThread)
+{
+ Assert(hThread == NIL_RTTHREAD); RT_NOREF_PV(hThread);
+#if RTLNX_VER_MIN(2,5,4)
+ return !!test_tsk_thread_flag(current, TIF_NEED_RESCHED);
+
+#elif RTLNX_VER_MIN(2,4,20)
+ return !!need_resched();
+
+#elif RTLNX_VER_MIN(2,1,110)
+ return current->need_resched != 0;
+
+#else
+ return need_resched != 0;
+#endif
+}
+RT_EXPORT_SYMBOL(RTThreadPreemptIsPending);
+
+
+RTDECL(bool) RTThreadPreemptIsPendingTrusty(void)
+{
+ /* yes, RTThreadPreemptIsPending is reliable. */
+ return true;
+}
+RT_EXPORT_SYMBOL(RTThreadPreemptIsPendingTrusty);
+
+
+RTDECL(bool) RTThreadPreemptIsPossible(void)
+{
+#ifdef IPRT_LNX_HAVE_PREEMPTION
+ return true; /* Yes, kernel preemption is possible. */
+#else
+ return false; /* No kernel preemption (or just CONFIG_PREEMPT_VOLUNTARY). */
+#endif
+}
+RT_EXPORT_SYMBOL(RTThreadPreemptIsPossible);
+
+
+RTDECL(void) RTThreadPreemptDisable(PRTTHREADPREEMPTSTATE pState)
+{
+#ifdef IPRT_LNX_HAVE_PREEMPTION
+ AssertPtr(pState);
+ Assert(pState->u32Reserved == 0);
+ pState->u32Reserved = 42;
+ preempt_disable();
+ RT_ASSERT_PREEMPT_CPUID_DISABLE(pState);
+
+#else /* !IPRT_LNX_HAVE_PREEMPTION */
+ int32_t c;
+ AssertPtr(pState);
+ Assert(pState->u32Reserved == 0);
+
+ /* Do our own accounting. */
+ c = ASMAtomicIncS32(&g_acPreemptDisabled[smp_processor_id()]);
+ AssertMsg(c > 0 && c < 32, ("%d\n", c));
+ pState->u32Reserved = c;
+ RT_ASSERT_PREEMPT_CPUID_DISABLE(pState);
+#endif
+}
+RT_EXPORT_SYMBOL(RTThreadPreemptDisable);
+
+
+RTDECL(void) RTThreadPreemptRestore(PRTTHREADPREEMPTSTATE pState)
+{
+#ifdef IPRT_LNX_HAVE_PREEMPTION
+ IPRT_LINUX_SAVE_EFL_AC(); /* paranoia */
+ AssertPtr(pState);
+ Assert(pState->u32Reserved == 42);
+ RT_ASSERT_PREEMPT_CPUID_RESTORE(pState);
+ preempt_enable();
+ IPRT_LINUX_RESTORE_EFL_ONLY_AC(); /* paranoia */
+
+#else
+ int32_t volatile *pc;
+ AssertPtr(pState);
+ AssertMsg(pState->u32Reserved > 0 && pState->u32Reserved < 32, ("%d\n", pState->u32Reserved));
+ RT_ASSERT_PREEMPT_CPUID_RESTORE(pState);
+
+ /* Do our own accounting. */
+ pc = &g_acPreemptDisabled[smp_processor_id()];
+ AssertMsg(pState->u32Reserved == (uint32_t)*pc, ("u32Reserved=%d *pc=%d \n", pState->u32Reserved, *pc));
+ ASMAtomicUoWriteS32(pc, pState->u32Reserved - 1);
+#endif
+ pState->u32Reserved = 0;
+}
+RT_EXPORT_SYMBOL(RTThreadPreemptRestore);
+
+
+RTDECL(bool) RTThreadIsInInterrupt(RTTHREAD hThread)
+{
+ Assert(hThread == NIL_RTTHREAD); NOREF(hThread);
+
+ return in_interrupt() != 0;
+}
+RT_EXPORT_SYMBOL(RTThreadIsInInterrupt);
+
+
+RTDECL(int) RTThreadQueryTerminationStatus(RTTHREAD hThread)
+{
+ struct task_struct *pTask = current;
+ AssertReturn(hThread == NIL_RTTHREAD, VERR_NOT_SUPPORTED);
+
+ /* Check out pending signals. ASSUMES we can get away w/o locking
+ anything because we're only reading the data. */
+ if (sigismember(&pTask->pending.signal, SIGKILL))
+ return VINF_THREAD_IS_TERMINATING;
+
+#if RTLNX_VER_MIN(2,5,34)
+ /* Check the pending signals shared with other threads in
+ the same process/group. ASSUME since we're alive that
+ the signal_struct won't be freed while we're looking
+ at it here... */
+ {
+# if RTLNX_VER_MIN(2,5,60)
+ struct signal_struct *pSignal = current->signal;
+# else
+ struct signal_struct *pSignal = current->sig;
+# endif
+ if ( pSignal
+ && sigismember(&pSignal->shared_pending.signal, SIGKILL))
+ return VINF_THREAD_IS_TERMINATING;
+ }
+#endif
+
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTThreadQueryTerminationStatus);
+
diff --git a/src/VBox/Runtime/r0drv/linux/thread2-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/thread2-r0drv-linux.c
new file mode 100644
index 00000000..16cebd7f
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/thread2-r0drv-linux.c
@@ -0,0 +1,243 @@
+/* $Id: thread2-r0drv-linux.c $ */
+/** @file
+ * IPRT - Threads (Part 2), Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/errcore.h>
+#include "internal/thread.h"
+
+#if RTLNX_VER_MIN(4,11,0)
+ #include <uapi/linux/sched/types.h>
+#endif /* >= KERNEL_VERSION(4, 11, 0) */
+
+RTDECL(RTTHREAD) RTThreadSelf(void)
+{
+ return rtThreadGetByNative((RTNATIVETHREAD)current);
+}
+
+
+DECLHIDDEN(int) rtThreadNativeInit(void)
+{
+ return VINF_SUCCESS;
+}
+
+
+DECLHIDDEN(int) rtThreadNativeSetPriority(PRTTHREADINT pThread, RTTHREADTYPE enmType)
+{
+#if RTLNX_VER_MIN(2,5,2)
+ /*
+ * Assignments are partially based on g_aTypesLinuxFree but
+ * scaled up in the high priority end.
+ *
+ * 5.9.0 - :
+ * The sched_set_normal interfaces does not really check the input,
+ * whereas sched_set_fifo & sched_set_fifo_low have fixed assignments.
+ * 2.6.11 - 5.9.0:
+ * Use sched_setscheduler to try effect FIFO scheduling
+ * for IO and TIMER threads, otherwise use set_user_nice.
+ * 2.5.2 - 5.9.0:
+ * Use set_user_nice to renice the thread.
+ */
+ int iNice = 0;
+# if RTLNX_VER_MAX(5,9,0)
+ int rc;
+# if RTLNX_VER_MIN(2,6,11)
+ int iSchedClass = SCHED_NORMAL;
+ struct sched_param Param = { .sched_priority = 0 };
+# endif
+# endif
+ switch (enmType)
+ {
+ case RTTHREADTYPE_INFREQUENT_POLLER:
+ iNice = +3;
+ break;
+
+ case RTTHREADTYPE_MAIN_HEAVY_WORKER:
+ iNice = +2;
+ break;
+
+ case RTTHREADTYPE_EMULATION:
+ iNice = +1;
+ break;
+
+ case RTTHREADTYPE_DEFAULT:
+ case RTTHREADTYPE_GUI:
+ case RTTHREADTYPE_MAIN_WORKER:
+ iNice = 0;
+ break;
+
+ case RTTHREADTYPE_VRDP_IO:
+ case RTTHREADTYPE_DEBUGGER:
+ iNice = -1;
+ break;
+
+ case RTTHREADTYPE_MSG_PUMP:
+ iNice = -2;
+ break;
+
+ case RTTHREADTYPE_IO:
+# if RTLNX_VER_MIN(5,9,0)
+ sched_set_fifo_low(current);
+ return VINF_SUCCESS;
+# else
+ iNice = -12;
+# if RTLNX_VER_MIN(2,6,11)
+ iSchedClass = SCHED_FIFO;
+ Param.sched_priority = 1; /* => prio=98; */
+# endif
+ break;
+# endif
+
+ case RTTHREADTYPE_TIMER:
+# if RTLNX_VER_MIN(5,9,0)
+ sched_set_fifo(current);
+ return VINF_SUCCESS;
+# else
+ iNice = -19;
+# if RTLNX_VER_MIN(2,6,11)
+ iSchedClass = SCHED_FIFO;
+ Param.sched_priority = MAX_RT_PRIO / 2; /* => prio=49 */
+# endif
+ break;
+# endif
+ default:
+ AssertMsgFailedReturn(("enmType=%d\n", enmType), VERR_INVALID_PARAMETER);
+ }
+
+# if RTLNX_VER_MIN(5,9,0)
+ /*
+ * We only get here for renice work.
+ */
+ sched_set_normal(current, iNice);
+
+# else /* < 5.9.0 */
+# if RTLNX_VER_MIN(2,6,11)
+ /*
+ * Try set scheduler parameters.
+ * Fall back on normal + nice if this fails for FIFO policy.*
+ */
+ rc = sched_setscheduler(current, iSchedClass, &Param);
+ if (rc)
+ {
+ Param.sched_priority = 0;
+ iSchedClass = SCHED_NORMAL;
+ rc = sched_setscheduler(current, iSchedClass, &Param);
+ }
+
+ /*
+ * Renice if using normal scheduling class.
+ */
+ if (iSchedClass == SCHED_NORMAL)
+# endif /* >= 2.6.11 */
+ set_user_nice(current, iNice);
+
+# endif /* < 5.9.0 */
+#else /* < 2.5.2 */
+ RT_NOREF_PV(enmType);
+#endif /* < 2.5.2 */
+ RT_NOREF_PV(pThread);
+ return VINF_SUCCESS;
+}
+
+
+DECLHIDDEN(int) rtThreadNativeAdopt(PRTTHREADINT pThread)
+{
+ RT_NOREF_PV(pThread);
+ return VERR_NOT_IMPLEMENTED;
+}
+
+
+DECLHIDDEN(void) rtThreadNativeWaitKludge(PRTTHREADINT pThread)
+{
+ /** @todo fix RTThreadWait/RTR0Term race on linux. */
+ RTThreadSleep(1); NOREF(pThread);
+}
+
+
+DECLHIDDEN(void) rtThreadNativeDestroy(PRTTHREADINT pThread)
+{
+ NOREF(pThread);
+}
+
+
+#if RTLNX_VER_MIN(2,6,4)
+/**
+ * Native kernel thread wrapper function.
+ *
+ * This will forward to rtThreadMain and do termination upon return.
+ *
+ * @param pvArg Pointer to the argument package.
+ */
+static int rtThreadNativeMain(void *pvArg)
+{
+ PRTTHREADINT pThread = (PRTTHREADINT)pvArg;
+
+ rtThreadMain(pThread, (RTNATIVETHREAD)current, &pThread->szName[0]);
+ return 0;
+}
+#endif
+
+
+DECLHIDDEN(int) rtThreadNativeCreate(PRTTHREADINT pThreadInt, PRTNATIVETHREAD pNativeThread)
+{
+#if RTLNX_VER_MIN(2,6,4)
+ struct task_struct *NativeThread;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ RT_ASSERT_PREEMPTIBLE();
+
+ NativeThread = kthread_run(rtThreadNativeMain, pThreadInt, "iprt-%s", pThreadInt->szName);
+
+ if (!IS_ERR(NativeThread))
+ {
+ *pNativeThread = (RTNATIVETHREAD)NativeThread;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_GENERAL_FAILURE;
+#else
+ return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
diff --git a/src/VBox/Runtime/r0drv/linux/threadctxhooks-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/threadctxhooks-r0drv-linux.c
new file mode 100644
index 00000000..a7b7d905
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/threadctxhooks-r0drv-linux.c
@@ -0,0 +1,341 @@
+/* $Id: threadctxhooks-r0drv-linux.c $ */
+/** @file
+ * IPRT - Thread Context Switching Hook, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2013-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/mem.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/errcore.h>
+#include <iprt/asm.h>
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h>
+#endif
+#include "internal/thread.h"
+
+
+/*
+ * Linux kernel 2.6.23 introduced preemption notifiers but RedHat 2.6.18 kernels
+ * got it backported.
+ */
+#if RTLNX_VER_MIN(2,6,18) && defined(CONFIG_PREEMPT_NOTIFIERS)
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+/**
+ * The internal hook object for linux.
+ */
+typedef struct RTTHREADCTXHOOKINT
+{
+ /** Magic value (RTTHREADCTXHOOKINT_MAGIC). */
+ uint32_t volatile u32Magic;
+ /** The thread handle (owner) for which the hook is registered. */
+ RTNATIVETHREAD hOwner;
+ /** The preemption notifier object. */
+ struct preempt_notifier LnxPreemptNotifier;
+ /** Whether the hook is enabled or not. If enabled, the LnxPreemptNotifier
+ * is linked into the owning thread's list of preemption callouts. */
+ bool fEnabled;
+ /** Pointer to the user callback. */
+ PFNRTTHREADCTXHOOK pfnCallback;
+ /** User argument passed to the callback. */
+ void *pvUser;
+ /** The linux callbacks. */
+ struct preempt_ops PreemptOps;
+#if RTLNX_VER_MIN(3,1,19) && defined(RT_ARCH_AMD64)
+ /** Starting with 3.1.19, the linux kernel doesn't restore kernel RFLAGS during
+ * task switch, so we have to do that ourselves. (x86 code is not affected.) */
+ RTCCUINTREG fSavedRFlags;
+#endif
+} RTTHREADCTXHOOKINT;
+typedef RTTHREADCTXHOOKINT *PRTTHREADCTXHOOKINT;
+
+
+/**
+ * Hook function for the thread schedule out event.
+ *
+ * @param pPreemptNotifier Pointer to the preempt_notifier struct.
+ * @param pNext Pointer to the task that is being scheduled
+ * instead of the current thread.
+ *
+ * @remarks Called with the rq (runqueue) lock held and with preemption and
+ * interrupts disabled!
+ */
+static void rtThreadCtxHooksLnxSchedOut(struct preempt_notifier *pPreemptNotifier, struct task_struct *pNext)
+{
+ PRTTHREADCTXHOOKINT pThis = RT_FROM_MEMBER(pPreemptNotifier, RTTHREADCTXHOOKINT, LnxPreemptNotifier);
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+ RTCCUINTREG fSavedEFlags = ASMGetFlags();
+ stac();
+#endif
+ RT_NOREF_PV(pNext);
+
+ AssertPtr(pThis);
+ AssertPtr(pThis->pfnCallback);
+ Assert(pThis->fEnabled);
+ Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+ pThis->pfnCallback(RTTHREADCTXEVENT_OUT, pThis->pvUser);
+
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+ ASMSetFlags(fSavedEFlags);
+# if RTLNX_VER_MIN(3,1,19) && defined(RT_ARCH_AMD64)
+ pThis->fSavedRFlags = fSavedEFlags;
+# endif
+#endif
+}
+
+
+/**
+ * Hook function for the thread schedule in event.
+ *
+ * @param pPreemptNotifier Pointer to the preempt_notifier struct.
+ * @param iCpu The CPU this thread is being scheduled on.
+ *
+ * @remarks Called without holding the rq (runqueue) lock and with preemption
+ * enabled!
+ * @todo r=bird: Preemption is of course disabled when it is called.
+ */
+static void rtThreadCtxHooksLnxSchedIn(struct preempt_notifier *pPreemptNotifier, int iCpu)
+{
+ PRTTHREADCTXHOOKINT pThis = RT_FROM_MEMBER(pPreemptNotifier, RTTHREADCTXHOOKINT, LnxPreemptNotifier);
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+ RTCCUINTREG fSavedEFlags = ASMGetFlags();
+ stac();
+#endif
+ RT_NOREF_PV(iCpu);
+
+ AssertPtr(pThis);
+ AssertPtr(pThis->pfnCallback);
+ Assert(pThis->fEnabled);
+
+ pThis->pfnCallback(RTTHREADCTXEVENT_IN, pThis->pvUser);
+
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# if RTLNX_VER_MIN(3,1,19) && defined(RT_ARCH_AMD64)
+ fSavedEFlags &= ~RT_BIT_64(18) /*X86_EFL_AC*/;
+ fSavedEFlags |= pThis->fSavedRFlags & RT_BIT_64(18) /*X86_EFL_AC*/;
+# endif
+ ASMSetFlags(fSavedEFlags);
+#endif
+}
+
+
+/**
+ * Worker function for RTThreadCtxHooks(Deregister|Release)().
+ *
+ * @param pThis Pointer to the internal thread-context object.
+ */
+DECLINLINE(void) rtThreadCtxHookDisable(PRTTHREADCTXHOOKINT pThis)
+{
+ Assert(pThis->PreemptOps.sched_out == rtThreadCtxHooksLnxSchedOut);
+ Assert(pThis->PreemptOps.sched_in == rtThreadCtxHooksLnxSchedIn);
+ preempt_disable();
+ preempt_notifier_unregister(&pThis->LnxPreemptNotifier);
+ pThis->fEnabled = false;
+ preempt_enable();
+}
+
+
+RTDECL(int) RTThreadCtxHookCreate(PRTTHREADCTXHOOK phCtxHook, uint32_t fFlags, PFNRTTHREADCTXHOOK pfnCallback, void *pvUser)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate input.
+ */
+ PRTTHREADCTXHOOKINT pThis;
+ Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+ AssertPtrReturn(pfnCallback, VERR_INVALID_POINTER);
+ AssertReturn(fFlags == 0, VERR_INVALID_FLAGS);
+
+ /*
+ * Allocate and initialize a new hook. We don't register it yet, just
+ * create it.
+ */
+ pThis = (PRTTHREADCTXHOOKINT)RTMemAllocZ(sizeof(*pThis));
+ if (RT_UNLIKELY(!pThis))
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_NO_MEMORY;
+ }
+ pThis->u32Magic = RTTHREADCTXHOOKINT_MAGIC;
+ pThis->hOwner = RTThreadNativeSelf();
+ pThis->fEnabled = false;
+ pThis->pfnCallback = pfnCallback;
+ pThis->pvUser = pvUser;
+ preempt_notifier_init(&pThis->LnxPreemptNotifier, &pThis->PreemptOps);
+ pThis->PreemptOps.sched_out = rtThreadCtxHooksLnxSchedOut;
+ pThis->PreemptOps.sched_in = rtThreadCtxHooksLnxSchedIn;
+
+#if RTLNX_VER_MIN(4,2,0)
+ preempt_notifier_inc();
+#endif
+
+ *phCtxHook = pThis;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTThreadCtxHookCreate);
+
+
+RTDECL(int ) RTThreadCtxHookDestroy(RTTHREADCTXHOOK hCtxHook)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate input.
+ */
+ PRTTHREADCTXHOOKINT pThis = hCtxHook;
+ if (pThis == NIL_RTTHREADCTXHOOK)
+ return VINF_SUCCESS;
+ AssertPtr(pThis);
+ AssertMsgReturn(pThis->u32Magic == RTTHREADCTXHOOKINT_MAGIC, ("pThis->u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis),
+ VERR_INVALID_HANDLE);
+ Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+ Assert(!pThis->fEnabled || pThis->hOwner == RTThreadNativeSelf());
+
+ /*
+ * If there's still a registered thread-context hook, deregister it now before destroying the object.
+ */
+ if (pThis->fEnabled)
+ {
+ Assert(pThis->hOwner == RTThreadNativeSelf());
+ rtThreadCtxHookDisable(pThis);
+ Assert(!pThis->fEnabled); /* paranoia */
+ }
+
+#if RTLNX_VER_MIN(4,2,0)
+ preempt_notifier_dec();
+#endif
+
+ ASMAtomicWriteU32(&pThis->u32Magic, ~RTTHREADCTXHOOKINT_MAGIC);
+ RTMemFree(pThis);
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTThreadCtxHookDestroy);
+
+
+RTDECL(int) RTThreadCtxHookEnable(RTTHREADCTXHOOK hCtxHook)
+{
+ /*
+ * Validate input.
+ */
+ PRTTHREADCTXHOOKINT pThis = hCtxHook;
+ AssertPtr(pThis);
+ AssertMsgReturn(pThis->u32Magic == RTTHREADCTXHOOKINT_MAGIC, ("pThis->u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis),
+ VERR_INVALID_HANDLE);
+ Assert(pThis->hOwner == RTThreadNativeSelf());
+ Assert(!pThis->fEnabled);
+ if (!pThis->fEnabled)
+ {
+ IPRT_LINUX_SAVE_EFL_AC();
+ Assert(pThis->PreemptOps.sched_out == rtThreadCtxHooksLnxSchedOut);
+ Assert(pThis->PreemptOps.sched_in == rtThreadCtxHooksLnxSchedIn);
+
+ /*
+ * Register the callback.
+ */
+ preempt_disable();
+ pThis->fEnabled = true;
+ preempt_notifier_register(&pThis->LnxPreemptNotifier);
+ preempt_enable();
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ }
+
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTThreadCtxHookEnable);
+
+
+RTDECL(int) RTThreadCtxHookDisable(RTTHREADCTXHOOK hCtxHook)
+{
+ /*
+ * Validate input.
+ */
+ PRTTHREADCTXHOOKINT pThis = hCtxHook;
+ if (pThis != NIL_RTTHREADCTXHOOK)
+ {
+ AssertPtr(pThis);
+ AssertMsgReturn(pThis->u32Magic == RTTHREADCTXHOOKINT_MAGIC, ("pThis->u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis),
+ VERR_INVALID_HANDLE);
+ Assert(pThis->hOwner == RTThreadNativeSelf());
+
+ /*
+ * Deregister the callback.
+ */
+ if (pThis->fEnabled)
+ {
+ IPRT_LINUX_SAVE_EFL_AC();
+ rtThreadCtxHookDisable(pThis);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ }
+ }
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTThreadCtxHookDisable);
+
+
+RTDECL(bool) RTThreadCtxHookIsEnabled(RTTHREADCTXHOOK hCtxHook)
+{
+ /*
+ * Validate input.
+ */
+ PRTTHREADCTXHOOKINT pThis = hCtxHook;
+ if (pThis == NIL_RTTHREADCTXHOOK)
+ return false;
+ AssertPtr(pThis);
+ AssertMsgReturn(pThis->u32Magic == RTTHREADCTXHOOKINT_MAGIC, ("pThis->u32Magic=%RX32 pThis=%p\n", pThis->u32Magic, pThis),
+ false);
+
+ return pThis->fEnabled;
+}
+RT_EXPORT_SYMBOL(RTThreadCtxHookIsEnabled);
+
+#else /* Not supported / Not needed */
+# include "../generic/threadctxhooks-r0drv-generic.cpp"
+#endif /* Not supported / Not needed */
+
diff --git a/src/VBox/Runtime/r0drv/linux/time-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/time-r0drv-linux.c
new file mode 100644
index 00000000..370afbf1
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/time-r0drv-linux.c
@@ -0,0 +1,221 @@
+/* $Id: time-r0drv-linux.c $ */
+/** @file
+ * IPRT - Time, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#define LOG_GROUP RTLOGGROUP_TIME
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+/* Make sure we have the setting functions we need for RTTimeNow: */
+#if RTLNX_VER_MAX(2,6,16)
+# define RTTIME_INCL_TIMEVAL
+#elif RTLNX_VER_MAX(3,17,0)
+# define RTTIME_INCL_TIMESPEC
+#endif
+#include <iprt/time.h>
+#include <iprt/asm.h>
+
+
+
+DECLINLINE(uint64_t) rtTimeGetSystemNanoTS(void)
+{
+#if RTLNX_VER_MIN(5,6,0)
+ /*
+ * Starting with kernel version 5.6-rc3 only 64-bit time interfaces
+ * are allowed in the kernel.
+ */
+ uint64_t u64;
+ struct timespec64 Ts = { 0, 0 };
+
+ ktime_get_ts64(&Ts);
+ u64 = Ts.tv_sec * RT_NS_1SEC_64 + Ts.tv_nsec;
+ return u64;
+
+#elif RTLNX_VER_MIN(2,6,16) /* This must match timer-r0drv-linux.c! */
+ /*
+ * Use ktime_get_ts, this is also what clock_gettime(CLOCK_MONOTONIC,) is using.
+ */
+ uint64_t u64;
+ struct timespec Ts = { 0, 0 };
+ ktime_get_ts(&Ts);
+ u64 = Ts.tv_sec * RT_NS_1SEC_64 + Ts.tv_nsec;
+ return u64;
+
+#elif RTLNX_VER_MIN(2,5,60)
+ /*
+ * Seems there is no way of getting to the exact source of
+ * sys_clock_gettime(CLOCK_MONOTONIC, &ts) here, I think. But
+ * 64-bit jiffies adjusted for the initial value should be pretty
+ * much the same I hope.
+ */
+ uint64_t u64 = get_jiffies_64();
+# ifdef INITIAL_JIFFIES
+ u64 += INITIAL_JIFFIES;
+# endif
+ u64 *= TICK_NSEC;
+ return u64;
+
+#else /* < 2.5.60 */
+# if BITS_PER_LONG >= 64
+ /*
+ * This is the same as above, except that there is no get_jiffies_64()
+ * here and we rely on long, and therefor jiffies, being 64-bit instead.
+ */
+ uint64_t u64 = jiffies;
+# ifdef INITIAL_JIFFIES
+ u64 += INITIAL_JIFFIES;
+# endif
+ u64 *= TICK_NSEC;
+ return u64;
+
+# else /* 32 bit jiffies */
+ /*
+ * We'll have to try track jiffy rollovers here or we'll be
+ * in trouble every time it flips.
+ *
+ * The high dword of the s_u64Last is the rollover count, the
+ * low dword is the previous jiffies. Updating is done by
+ * atomic compare & exchange of course.
+ */
+ static uint64_t volatile s_u64Last = 0;
+ uint64_t u64;
+
+ for (;;)
+ {
+ uint64_t u64NewLast;
+ int32_t iDelta;
+ uint32_t cRollovers;
+ uint32_t u32LastJiffies;
+
+ /* sample the values */
+ unsigned long ulNow = jiffies;
+ uint64_t u64Last = s_u64Last;
+ if (ulNow != jiffies)
+ continue; /* try again */
+# ifdef INITIAL_JIFFIES
+ ulNow += INITIAL_JIFFIES;
+# endif
+
+ u32LastJiffies = (uint32_t)u64Last;
+ cRollovers = u64Last >> 32;
+
+ /*
+ * Check for rollover and update the static last value.
+ *
+ * We have to make sure we update it successfully to rule out
+ * an underrun because of racing someone.
+ */
+ iDelta = ulNow - u32LastJiffies;
+ if (iDelta < 0)
+ {
+ cRollovers++;
+ u64NewLast = RT_MAKE_U64(ulNow, cRollovers);
+ if (!ASMAtomicCmpXchgU64(&s_u64Last, u64NewLast, u64Last))
+ continue; /* race, try again */
+ }
+ else
+ {
+ u64NewLast = RT_MAKE_U64(ulNow, cRollovers);
+ ASMAtomicCmpXchgU64(&s_u64Last, u64NewLast, u64Last);
+ }
+
+ /* calculate the return value */
+ u64 = ulNow;
+ u64 *= TICK_NSEC;
+ u64 += cRollovers * (_4G * TICK_NSEC);
+ break;
+ }
+
+ return u64;
+# endif /* 32 bit jiffies */
+#endif /* < 2.5.60 */
+}
+
+
+RTDECL(uint64_t) RTTimeNanoTS(void)
+{
+ return rtTimeGetSystemNanoTS();
+}
+RT_EXPORT_SYMBOL(RTTimeNanoTS);
+
+
+RTDECL(uint64_t) RTTimeMilliTS(void)
+{
+ return rtTimeGetSystemNanoTS() / RT_NS_1MS;
+}
+RT_EXPORT_SYMBOL(RTTimeMilliTS);
+
+
+RTDECL(uint64_t) RTTimeSystemNanoTS(void)
+{
+ return rtTimeGetSystemNanoTS();
+}
+RT_EXPORT_SYMBOL(RTTimeSystemNanoTS);
+
+
+RTDECL(uint64_t) RTTimeSystemMilliTS(void)
+{
+ return rtTimeGetSystemNanoTS() / RT_NS_1MS;
+}
+RT_EXPORT_SYMBOL(RTTimeSystemMilliTS);
+
+
+RTDECL(PRTTIMESPEC) RTTimeNow(PRTTIMESPEC pTime)
+{
+ IPRT_LINUX_SAVE_EFL_AC();
+#if RTLNX_VER_MIN(3,17,0)
+ struct timespec64 Ts;
+ ktime_get_real_ts64(&Ts); /* ktime_get_real_ts64 was added as a macro in 3.17, function since 4.18. */
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return RTTimeSpecSetTimespec64(pTime, &Ts);
+
+#elif RTLNX_VER_MIN(2,6,16)
+ struct timespec Ts;
+ ktime_get_real_ts(&Ts); /* ktime_get_real_ts was removed in Linux 4.20. */
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return RTTimeSpecSetTimespec(pTime, &Ts);
+
+#else /* < 2.6.16 */
+ struct timeval Tv;
+ do_gettimeofday(&Tv);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return RTTimeSpecSetTimeval(pTime, &Tv);
+#endif
+}
+RT_EXPORT_SYMBOL(RTTimeNow);
+
diff --git a/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c b/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c
new file mode 100644
index 00000000..68886a8c
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c
@@ -0,0 +1,1739 @@
+/* $Id: timer-r0drv-linux.c $ */
+/** @file
+ * IPRT - Timers, Ring-0 Driver, Linux.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+
+/*********************************************************************************************************************************
+* Header Files *
+*********************************************************************************************************************************/
+#include "the-linux-kernel.h"
+#include "internal/iprt.h"
+
+#include <iprt/timer.h>
+#include <iprt/time.h>
+#include <iprt/mp.h>
+#include <iprt/cpuset.h>
+#include <iprt/spinlock.h>
+#include <iprt/err.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+
+#include "internal/magics.h"
+
+/** @def RTTIMER_LINUX_WITH_HRTIMER
+ * Whether to use high resolution timers. */
+#if !defined(RTTIMER_LINUX_WITH_HRTIMER) \
+ && defined(IPRT_LINUX_HAS_HRTIMER)
+# define RTTIMER_LINUX_WITH_HRTIMER
+#endif
+
+#if RTLNX_VER_MAX(2,6,31)
+# define mod_timer_pinned mod_timer
+# define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS
+#endif
+
+
+/*********************************************************************************************************************************
+* Structures and Typedefs *
+*********************************************************************************************************************************/
+/**
+ * Timer state machine.
+ *
+ * This is used to try handle the issues with MP events and
+ * timers that runs on all CPUs. It's relatively nasty :-/
+ */
+typedef enum RTTIMERLNXSTATE
+{
+ /** Stopped. */
+ RTTIMERLNXSTATE_STOPPED = 0,
+ /** Transient state; next ACTIVE. */
+ RTTIMERLNXSTATE_STARTING,
+ /** Transient state; next ACTIVE. (not really necessary) */
+ RTTIMERLNXSTATE_MP_STARTING,
+ /** Active. */
+ RTTIMERLNXSTATE_ACTIVE,
+ /** Active and in callback; next ACTIVE, STOPPED or CALLBACK_DESTROYING. */
+ RTTIMERLNXSTATE_CALLBACK,
+ /** Stopped while in the callback; next STOPPED. */
+ RTTIMERLNXSTATE_CB_STOPPING,
+ /** Restarted while in the callback; next ACTIVE, STOPPED, DESTROYING. */
+ RTTIMERLNXSTATE_CB_RESTARTING,
+ /** The callback shall destroy the timer; next STOPPED. */
+ RTTIMERLNXSTATE_CB_DESTROYING,
+ /** Transient state; next STOPPED. */
+ RTTIMERLNXSTATE_STOPPING,
+ /** Transient state; next STOPPED. */
+ RTTIMERLNXSTATE_MP_STOPPING,
+ /** The usual 32-bit hack. */
+ RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
+} RTTIMERLNXSTATE;
+
+
+/**
+ * A Linux sub-timer.
+ */
+typedef struct RTTIMERLNXSUBTIMER
+{
+ /** Timer specific data. */
+ union
+ {
+#if defined(RTTIMER_LINUX_WITH_HRTIMER)
+ /** High resolution timer. */
+ struct
+ {
+ /** The linux timer structure. */
+ struct hrtimer LnxTimer;
+ } Hr;
+#endif
+ /** Standard timer. */
+ struct
+ {
+ /** The linux timer structure. */
+ struct timer_list LnxTimer;
+ /** The start of the current run (ns).
+ * This is used to calculate when the timer ought to fire the next time. */
+ uint64_t u64NextTS;
+ /** When the timer was started. */
+ uint64_t nsStartTS;
+ /** The u64NextTS in jiffies. */
+ unsigned long ulNextJiffies;
+ /** Set when starting or changing the timer so that u64StartTs
+ * and u64NextTS gets reinitialized (eliminating some jitter). */
+ bool volatile fFirstAfterChg;
+ } Std;
+ } u;
+ /** The current tick number. */
+ uint64_t iTick;
+ /** Restart the single shot timer at this specific time.
+ * Used when a single shot timer is restarted from the callback. */
+ uint64_t volatile uNsRestartAt;
+ /** Pointer to the parent timer. */
+ PRTTIMER pParent;
+ /** The current sub-timer state. */
+ RTTIMERLNXSTATE volatile enmState;
+} RTTIMERLNXSUBTIMER;
+/** Pointer to a linux sub-timer. */
+typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
+
+
+/**
+ * The internal representation of an Linux timer handle.
+ */
+typedef struct RTTIMER
+{
+ /** Magic.
+ * This is RTTIMER_MAGIC, but changes to something else before the timer
+ * is destroyed to indicate clearly that thread should exit. */
+ uint32_t volatile u32Magic;
+ /** Spinlock synchronizing the fSuspended and MP event handling.
+ * This is NIL_RTSPINLOCK if cCpus == 1. */
+ RTSPINLOCK hSpinlock;
+ /** Flag indicating that the timer is suspended. */
+ bool volatile fSuspended;
+ /** Whether the timer must run on one specific CPU or not. */
+ bool fSpecificCpu;
+#ifdef CONFIG_SMP
+ /** Whether the timer must run on all CPUs or not. */
+ bool fAllCpus;
+#endif /* else: All -> specific on non-SMP kernels */
+ /** Whether it is a high resolution timer or a standard one. */
+ bool fHighRes;
+ /** The id of the CPU it must run on if fSpecificCpu is set. */
+ RTCPUID idCpu;
+ /** The number of CPUs this timer should run on. */
+ RTCPUID cCpus;
+ /** Callback. */
+ PFNRTTIMER pfnTimer;
+ /** User argument. */
+ void *pvUser;
+ /** The timer interval. 0 if one-shot. */
+ uint64_t volatile u64NanoInterval;
+ /** This is set to the number of jiffies between ticks if the interval is
+ * an exact number of jiffies. (Standard timers only.) */
+ unsigned long volatile cJiffies;
+ /** The change interval spinlock for standard timers only. */
+ spinlock_t ChgIntLock;
+ /** Workqueue item for delayed destruction. */
+ RTR0LNXWORKQUEUEITEM DtorWorkqueueItem;
+ /** Sub-timers.
+ * Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
+ * an entry for all possible cpus. In that case the index will be the same as
+ * for the RTCpuSet. */
+ RTTIMERLNXSUBTIMER aSubTimers[1];
+} RTTIMER;
+
+
+/**
+ * A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
+ */
+typedef struct RTTIMERLINUXSTARTONCPUARGS
+{
+ /** The current time (RTTimeSystemNanoTS). */
+ uint64_t u64Now;
+ /** When to start firing (delta). */
+ uint64_t u64First;
+} RTTIMERLINUXSTARTONCPUARGS;
+/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
+typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
+
+
+/*********************************************************************************************************************************
+* Internal Functions *
+*********************************************************************************************************************************/
+#ifdef CONFIG_SMP
+static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser);
+#endif
+
+#if 0
+#define DEBUG_HACKING
+#include <iprt/string.h>
+#include <iprt/asm-amd64-x86.h>
+static void myLogBackdoorPrintf(const char *pszFormat, ...)
+{
+ char szTmp[256];
+ va_list args;
+ size_t cb;
+
+ cb = RTStrPrintf(szTmp, sizeof(szTmp) - 10, "%d: ", RTMpCpuId());
+ va_start(args, pszFormat);
+ cb += RTStrPrintfV(&szTmp[cb], sizeof(szTmp) - cb, pszFormat, args);
+ va_end(args);
+
+ ASMOutStrU8(0x504, (uint8_t *)&szTmp[0], cb);
+}
+# define RTAssertMsg1Weak(pszExpr, uLine, pszFile, pszFunction) \
+ myLogBackdoorPrintf("\n!!Guest Assertion failed!!\n%s(%d) %s\n%s\n", uLine, pszFile, pszFunction, (pszExpr))
+# define RTAssertMsg2Weak myLogBackdoorPrintf
+# define RTTIMERLNX_LOG(a) myLogBackdoorPrintf a
+#else
+# define RTTIMERLNX_LOG(a) do { } while (0)
+#endif
+
+/**
+ * Sets the state.
+ */
+DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
+{
+#ifdef DEBUG_HACKING
+ RTTIMERLNX_LOG(("set %d -> %d\n", *penmState, enmNewState));
+#endif
+ ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
+}
+
+
+/**
+ * Sets the state if it has a certain value.
+ *
+ * @return true if xchg was done.
+ * @return false if xchg wasn't done.
+ */
+#ifdef DEBUG_HACKING
+#define rtTimerLnxCmpXchgState(penmState, enmNewState, enmCurState) rtTimerLnxCmpXchgStateDebug(penmState, enmNewState, enmCurState, __LINE__)
+static bool rtTimerLnxCmpXchgStateDebug(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
+ RTTIMERLNXSTATE enmCurState, uint32_t uLine)
+{
+ RTTIMERLNXSTATE enmOldState = enmCurState;
+ bool fRc = ASMAtomicCmpXchgExU32((uint32_t volatile *)penmState, enmNewState, enmCurState, (uint32_t *)&enmOldState);
+ RTTIMERLNX_LOG(("cxg %d -> %d - %d at %u\n", enmOldState, enmNewState, fRc, uLine));
+ return fRc;
+}
+#else
+DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
+ RTTIMERLNXSTATE enmCurState)
+{
+ return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
+}
+#endif
+
+
+/**
+ * Gets the state.
+ */
+DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
+{
+ return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
+}
+
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+
+/**
+ * Converts a nano second time stamp to ktime_t.
+ *
+ * ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
+ *
+ * @returns ktime_t.
+ * @param cNanoSecs Nanoseconds.
+ */
+DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
+{
+ /* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
+ return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
+}
+
+/**
+ * Converts ktime_t to a nano second time stamp.
+ *
+ * ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
+ *
+ * @returns nano second time stamp.
+ * @param Kt ktime_t.
+ */
+DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
+{
+ return ktime_to_ns(Kt);
+}
+
+#endif /* RTTIMER_LINUX_WITH_HRTIMER */
+
+/**
+ * Converts a nano second interval to jiffies.
+ *
+ * @returns Jiffies.
+ * @param cNanoSecs Nanoseconds.
+ */
+DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
+{
+ /* this can be made even better... */
+ if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
+ return MAX_JIFFY_OFFSET;
+# if ARCH_BITS == 32
+ if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
+ return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
+# endif
+ return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
+}
+
+
+/**
+ * Starts a sub-timer (RTTimerStart).
+ *
+ * @param pSubTimer The sub-timer to start.
+ * @param u64Now The current timestamp (RTTimeSystemNanoTS()).
+ * @param u64First The interval from u64Now to the first time the timer should fire.
+ * @param fPinned true = timer pinned to a specific CPU,
+ * false = timer can migrate between CPUs
+ * @param fHighRes Whether the user requested a high resolution timer or not.
+ * @param enmOldState The old timer state.
+ */
+static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First,
+ bool fPinned, bool fHighRes)
+{
+ /*
+ * Calc when it should start firing.
+ */
+ uint64_t u64NextTS = u64Now + u64First;
+ if (!fHighRes)
+ {
+ pSubTimer->u.Std.u64NextTS = u64NextTS;
+ pSubTimer->u.Std.nsStartTS = u64NextTS;
+ }
+ RTTIMERLNX_LOG(("startsubtimer %p\n", pSubTimer->pParent));
+
+ pSubTimer->iTick = 0;
+
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+ if (fHighRes)
+ hrtimer_start(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(u64NextTS),
+ fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS);
+ else
+#endif
+ {
+ unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
+ pSubTimer->u.Std.ulNextJiffies = jiffies + cJiffies;
+ pSubTimer->u.Std.fFirstAfterChg = true;
+#ifdef CONFIG_SMP
+ if (fPinned)
+ {
+# if RTLNX_VER_MIN(4,8,0)
+ mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
+# else
+ mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
+# endif
+ }
+ else
+#endif
+ mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
+ }
+
+ /* Be a bit careful here since we could be racing the callback. */
+ if (!rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_STARTING))
+ rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_MP_STARTING);
+}
+
+
+/**
+ * Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
+ *
+ * The caller has already changed the state, so we will not be in a callback
+ * situation wrt to the calling thread.
+ *
+ * @param pSubTimer The sub-timer.
+ * @param fHighRes Whether the user requested a high resolution timer or not.
+ */
+static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, bool fHighRes)
+{
+ RTTIMERLNX_LOG(("stopsubtimer %p %d\n", pSubTimer->pParent, fHighRes));
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+ if (fHighRes)
+ {
+ /* There is no equivalent to del_timer in the hrtimer API,
+ hrtimer_cancel() == del_timer_sync(). Just like the WARN_ON in
+ del_timer_sync() asserts, waiting for a timer callback to complete
+ is deadlock prone, so don't do it. */
+ int rc = hrtimer_try_to_cancel(&pSubTimer->u.Hr.LnxTimer);
+ if (rc < 0)
+ {
+ hrtimer_start(&pSubTimer->u.Hr.LnxTimer, ktime_set(KTIME_SEC_MAX, 0), HRTIMER_MODE_ABS);
+ hrtimer_try_to_cancel(&pSubTimer->u.Hr.LnxTimer);
+ }
+ }
+ else
+#endif
+ del_timer(&pSubTimer->u.Std.LnxTimer);
+
+ rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
+}
+
+
+/**
+ * Used by RTTimerDestroy and rtTimerLnxCallbackDestroy to do the actual work.
+ *
+ * @param pTimer The timer in question.
+ */
+static void rtTimerLnxDestroyIt(PRTTIMER pTimer)
+{
+ RTSPINLOCK hSpinlock = pTimer->hSpinlock;
+ RTCPUID iCpu;
+ Assert(pTimer->fSuspended);
+ RTTIMERLNX_LOG(("destroyit %p\n", pTimer));
+
+ /*
+ * Remove the MP notifications first because it'll reduce the risk of
+ * us overtaking any MP event that might theoretically be racing us here.
+ */
+#ifdef CONFIG_SMP
+ if ( pTimer->cCpus > 1
+ && hSpinlock != NIL_RTSPINLOCK)
+ {
+ int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
+ AssertRC(rc);
+ }
+#endif /* CONFIG_SMP */
+
+ /*
+ * Invalidate the handle.
+ */
+ ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
+
+ /*
+ * Make sure all timers have stopped executing since we're stopping them in
+ * an asynchronous manner up in rtTimerLnxStopSubTimer.
+ */
+ iCpu = pTimer->cCpus;
+ while (iCpu-- > 0)
+ {
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+ if (pTimer->fHighRes)
+ hrtimer_cancel(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer);
+ else
+#endif
+ del_timer_sync(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
+ }
+
+ /*
+ * Finally, free the resources.
+ */
+ RTMemFreeEx(pTimer, RT_UOFFSETOF_DYN(RTTIMER, aSubTimers[pTimer->cCpus]));
+ if (hSpinlock != NIL_RTSPINLOCK)
+ RTSpinlockDestroy(hSpinlock);
+}
+
+
+/**
+ * Workqueue callback (no DECLCALLBACK!) for deferred destruction.
+ *
+ * @param pWork Pointer to the DtorWorkqueueItem member of our timer
+ * structure.
+ */
+static void rtTimerLnxDestroyDeferred(RTR0LNXWORKQUEUEITEM *pWork)
+{
+ PRTTIMER pTimer = RT_FROM_MEMBER(pWork, RTTIMER, DtorWorkqueueItem);
+ rtTimerLnxDestroyIt(pTimer);
+}
+
+
+/**
+ * Called when the timer was destroyed by the callback function.
+ *
+ * @param pTimer The timer.
+ * @param pSubTimer The sub-timer which we're handling, the state of this
+ * will be RTTIMERLNXSTATE_CALLBACK_DESTROYING.
+ */
+static void rtTimerLnxCallbackDestroy(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
+{
+ /*
+ * If it's an omni timer, the last dude does the destroying.
+ */
+ if (pTimer->cCpus > 1)
+ {
+ uint32_t iCpu = pTimer->cCpus;
+ RTSpinlockAcquire(pTimer->hSpinlock);
+
+ Assert(pSubTimer->enmState == RTTIMERLNXSTATE_CB_DESTROYING);
+ rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
+
+ while (iCpu-- > 0)
+ if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
+ {
+ RTSpinlockRelease(pTimer->hSpinlock);
+ return;
+ }
+
+ RTSpinlockRelease(pTimer->hSpinlock);
+ }
+
+ /*
+ * Destroying a timer from the callback is unsafe since the callout code
+ * might be touching the timer structure upon return (hrtimer does!). So,
+ * we have to defer the actual destruction to the IRPT workqueue.
+ */
+ rtR0LnxWorkqueuePush(&pTimer->DtorWorkqueueItem, rtTimerLnxDestroyDeferred);
+}
+
+
+#ifdef CONFIG_SMP
+/**
+ * Deal with a sub-timer that has migrated.
+ *
+ * @param pTimer The timer.
+ * @param pSubTimer The sub-timer.
+ */
+static void rtTimerLnxCallbackHandleMigration(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
+{
+ RTTIMERLNXSTATE enmState;
+ if (pTimer->cCpus > 1)
+ RTSpinlockAcquire(pTimer->hSpinlock);
+
+ do
+ {
+ enmState = rtTimerLnxGetState(&pSubTimer->enmState);
+ switch (enmState)
+ {
+ case RTTIMERLNXSTATE_STOPPING:
+ case RTTIMERLNXSTATE_MP_STOPPING:
+ enmState = RTTIMERLNXSTATE_STOPPED;
+ RT_FALL_THRU();
+ case RTTIMERLNXSTATE_STOPPED:
+ break;
+
+ default:
+ AssertMsgFailed(("%d\n", enmState));
+ RT_FALL_THRU();
+ case RTTIMERLNXSTATE_STARTING:
+ case RTTIMERLNXSTATE_MP_STARTING:
+ case RTTIMERLNXSTATE_ACTIVE:
+ case RTTIMERLNXSTATE_CALLBACK:
+ case RTTIMERLNXSTATE_CB_STOPPING:
+ case RTTIMERLNXSTATE_CB_RESTARTING:
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, enmState))
+ enmState = RTTIMERLNXSTATE_STOPPED;
+ break;
+
+ case RTTIMERLNXSTATE_CB_DESTROYING:
+ {
+ if (pTimer->cCpus > 1)
+ RTSpinlockRelease(pTimer->hSpinlock);
+
+ rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
+ return;
+ }
+ }
+ } while (enmState != RTTIMERLNXSTATE_STOPPED);
+
+ if (pTimer->cCpus > 1)
+ RTSpinlockRelease(pTimer->hSpinlock);
+}
+#endif /* CONFIG_SMP */
+
+
+/**
+ * The slow path of rtTimerLnxChangeToCallbackState.
+ *
+ * @returns true if changed successfully, false if not.
+ * @param pSubTimer The sub-timer.
+ */
+static bool rtTimerLnxChangeToCallbackStateSlow(PRTTIMERLNXSUBTIMER pSubTimer)
+{
+ for (;;)
+ {
+ RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
+ switch (enmState)
+ {
+ case RTTIMERLNXSTATE_ACTIVE:
+ case RTTIMERLNXSTATE_STARTING:
+ case RTTIMERLNXSTATE_MP_STARTING:
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, enmState))
+ return true;
+ break;
+
+ case RTTIMERLNXSTATE_CALLBACK:
+ case RTTIMERLNXSTATE_CB_STOPPING:
+ case RTTIMERLNXSTATE_CB_RESTARTING:
+ case RTTIMERLNXSTATE_CB_DESTROYING:
+ AssertMsgFailed(("%d\n", enmState)); RT_FALL_THRU();
+ default:
+ return false;
+ }
+ ASMNopPause();
+ }
+}
+
+
+/**
+ * Tries to change the sub-timer state to 'callback'.
+ *
+ * @returns true if changed successfully, false if not.
+ * @param pSubTimer The sub-timer.
+ */
+DECLINLINE(bool) rtTimerLnxChangeToCallbackState(PRTTIMERLNXSUBTIMER pSubTimer)
+{
+ if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, RTTIMERLNXSTATE_ACTIVE)))
+ return true;
+ return rtTimerLnxChangeToCallbackStateSlow(pSubTimer);
+}
+
+
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+/**
+ * Timer callback function for high resolution timers.
+ *
+ * @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a
+ * one-shot or interval timer.
+ * @param pHrTimer Pointer to the sub-timer structure.
+ */
+static enum hrtimer_restart rtTimerLinuxHrCallback(struct hrtimer *pHrTimer)
+{
+ PRTTIMERLNXSUBTIMER pSubTimer = RT_FROM_MEMBER(pHrTimer, RTTIMERLNXSUBTIMER, u.Hr.LnxTimer);
+ PRTTIMER pTimer = pSubTimer->pParent;
+
+
+ RTTIMERLNX_LOG(("hrcallback %p\n", pTimer));
+ if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
+ return HRTIMER_NORESTART;
+
+#ifdef CONFIG_SMP
+ /*
+ * Check for unwanted migration.
+ */
+ if (pTimer->fAllCpus || pTimer->fSpecificCpu)
+ {
+ RTCPUID idCpu = RTMpCpuId();
+ if (RT_UNLIKELY( pTimer->fAllCpus
+ ? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu
+ : pTimer->idCpu != idCpu))
+ {
+ rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
+ return HRTIMER_NORESTART;
+ }
+ }
+#endif
+
+ if (pTimer->u64NanoInterval)
+ {
+ /*
+ * Periodic timer, run it and update the native timer afterwards so
+ * we can handle RTTimerStop and RTTimerChangeInterval from the
+ * callback as well as a racing control thread.
+ */
+ pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
+ hrtimer_add_expires_ns(&pSubTimer->u.Hr.LnxTimer, ASMAtomicReadU64(&pTimer->u64NanoInterval));
+ if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
+ return HRTIMER_RESTART;
+ }
+ else
+ {
+ /*
+ * One shot timer (no omni), stop it before dispatching it.
+ * Allow RTTimerStart as well as RTTimerDestroy to be called from
+ * the callback.
+ */
+ ASMAtomicWriteBool(&pTimer->fSuspended, true);
+ pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
+ if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
+ return HRTIMER_NORESTART;
+ }
+
+ /*
+ * Some state change occurred while we were in the callback routine.
+ */
+ for (;;)
+ {
+ RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
+ switch (enmState)
+ {
+ case RTTIMERLNXSTATE_CB_DESTROYING:
+ rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
+ return HRTIMER_NORESTART;
+
+ case RTTIMERLNXSTATE_CB_STOPPING:
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
+ return HRTIMER_NORESTART;
+ break;
+
+ case RTTIMERLNXSTATE_CB_RESTARTING:
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
+ {
+ pSubTimer->iTick = 0;
+ hrtimer_set_expires(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(pSubTimer->uNsRestartAt));
+ return HRTIMER_RESTART;
+ }
+ break;
+
+ default:
+ AssertMsgFailed(("%d\n", enmState));
+ return HRTIMER_NORESTART;
+ }
+ ASMNopPause();
+ }
+}
+#endif /* RTTIMER_LINUX_WITH_HRTIMER */
+
+
+#if RTLNX_VER_MIN(4,15,0)
+/**
+ * Timer callback function for standard timers.
+ *
+ * @param pLnxTimer Pointer to the Linux timer structure.
+ */
+static void rtTimerLinuxStdCallback(struct timer_list *pLnxTimer)
+{
+ PRTTIMERLNXSUBTIMER pSubTimer = from_timer(pSubTimer, pLnxTimer, u.Std.LnxTimer);
+#else
+/**
+ * Timer callback function for standard timers.
+ *
+ * @param ulUser Address of the sub-timer structure.
+ */
+static void rtTimerLinuxStdCallback(unsigned long ulUser)
+{
+ PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
+#endif
+ PRTTIMER pTimer = pSubTimer->pParent;
+
+ RTTIMERLNX_LOG(("stdcallback %p\n", pTimer));
+ if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
+ return;
+
+#ifdef CONFIG_SMP
+ /*
+ * Check for unwanted migration.
+ */
+ if (pTimer->fAllCpus || pTimer->fSpecificCpu)
+ {
+ RTCPUID idCpu = RTMpCpuId();
+ if (RT_UNLIKELY( pTimer->fAllCpus
+ ? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu
+ : pTimer->idCpu != idCpu))
+ {
+ rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
+ return;
+ }
+ }
+#endif
+
+ if (pTimer->u64NanoInterval)
+ {
+ /*
+ * Interval timer, calculate the next timeout.
+ *
+ * The first time around, we'll re-adjust the u.Std.u64NextTS to
+ * try prevent some jittering if we were started at a bad time.
+ */
+ const uint64_t iTick = ++pSubTimer->iTick;
+ unsigned long uCurJiffies = jiffies;
+ unsigned long ulNextJiffies;
+ uint64_t u64NanoInterval;
+ unsigned long cJiffies;
+ unsigned long flFlags;
+
+ spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
+ u64NanoInterval = pTimer->u64NanoInterval;
+ cJiffies = pTimer->cJiffies;
+ if (RT_UNLIKELY(pSubTimer->u.Std.fFirstAfterChg))
+ {
+ pSubTimer->u.Std.fFirstAfterChg = false;
+ pSubTimer->u.Std.u64NextTS = RTTimeSystemNanoTS();
+ pSubTimer->u.Std.nsStartTS = pSubTimer->u.Std.u64NextTS - u64NanoInterval * (iTick - 1);
+ pSubTimer->u.Std.ulNextJiffies = uCurJiffies = jiffies;
+ }
+ spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
+
+ pSubTimer->u.Std.u64NextTS += u64NanoInterval;
+ if (cJiffies)
+ {
+ ulNextJiffies = pSubTimer->u.Std.ulNextJiffies + cJiffies;
+ pSubTimer->u.Std.ulNextJiffies = ulNextJiffies;
+ if (time_after_eq(ulNextJiffies, uCurJiffies))
+ { /* likely */ }
+ else
+ {
+ unsigned long cJiffiesBehind = uCurJiffies - ulNextJiffies;
+ ulNextJiffies = uCurJiffies + cJiffies / 2;
+ if (cJiffiesBehind >= HZ / 4) /* Conside if we're lagging too far behind. Screw the u64NextTS member. */
+ pSubTimer->u.Std.ulNextJiffies = ulNextJiffies;
+ /*else: Don't update u.Std.ulNextJiffies so we can continue catching up in the next tick. */
+ }
+ }
+ else
+ {
+ const uint64_t u64NanoTS = RTTimeSystemNanoTS();
+ const int64_t cNsBehind = u64NanoTS - pSubTimer->u.Std.u64NextTS;
+ if (cNsBehind <= 0)
+ ulNextJiffies = uCurJiffies + rtTimerLnxNanoToJiffies(pSubTimer->u.Std.u64NextTS - u64NanoTS);
+ else if (u64NanoInterval >= RT_NS_1SEC_64 * 2 / HZ)
+ {
+ ulNextJiffies = uCurJiffies + rtTimerLnxNanoToJiffies(u64NanoInterval / 2);
+ if (cNsBehind >= RT_NS_1SEC_64 / HZ / 4) /* Conside if we're lagging too far behind. */
+ pSubTimer->u.Std.u64NextTS = u64NanoTS + u64NanoInterval / 2;
+ }
+ else
+ {
+ ulNextJiffies = uCurJiffies + 1;
+ if (cNsBehind >= RT_NS_1SEC_64 / HZ / 4) /* Conside if we're lagging too far behind. */
+ pSubTimer->u.Std.u64NextTS = u64NanoTS + RT_NS_1SEC_64 / HZ;
+ }
+ pSubTimer->u.Std.ulNextJiffies = ulNextJiffies;
+ }
+
+ /*
+ * Run the timer and re-arm it unless the state changed .
+ * .
+ * We must re-arm it afterwards as we're not in a position to undo this .
+ * operation if for instance someone stopped or destroyed us while we .
+ * were in the callback. (Linux takes care of any races here.)
+ */
+ pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
+ if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
+ {
+#ifdef CONFIG_SMP
+ if (pTimer->fSpecificCpu || pTimer->fAllCpus)
+ {
+# if RTLNX_VER_MIN(4,8,0)
+ mod_timer(&pSubTimer->u.Std.LnxTimer, ulNextJiffies);
+# else
+ mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, ulNextJiffies);
+# endif
+ }
+ else
+#endif
+ mod_timer(&pSubTimer->u.Std.LnxTimer, ulNextJiffies);
+ return;
+ }
+ }
+ else
+ {
+ /*
+ * One shot timer, stop it before dispatching it.
+ * Allow RTTimerStart as well as RTTimerDestroy to be called from
+ * the callback.
+ */
+ ASMAtomicWriteBool(&pTimer->fSuspended, true);
+ pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
+ if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
+ return;
+ }
+
+ /*
+ * Some state change occurred while we were in the callback routine.
+ */
+ for (;;)
+ {
+ RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
+ switch (enmState)
+ {
+ case RTTIMERLNXSTATE_CB_DESTROYING:
+ rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
+ return;
+
+ case RTTIMERLNXSTATE_CB_STOPPING:
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
+ return;
+ break;
+
+ case RTTIMERLNXSTATE_CB_RESTARTING:
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
+ {
+ uint64_t u64NanoTS;
+ uint64_t u64NextTS;
+ unsigned long flFlags;
+
+ spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
+ u64NextTS = pSubTimer->uNsRestartAt;
+ u64NanoTS = RTTimeSystemNanoTS();
+ pSubTimer->iTick = 0;
+ pSubTimer->u.Std.u64NextTS = u64NextTS;
+ pSubTimer->u.Std.fFirstAfterChg = true;
+ pSubTimer->u.Std.ulNextJiffies = u64NextTS > u64NanoTS
+ ? jiffies + rtTimerLnxNanoToJiffies(u64NextTS - u64NanoTS)
+ : jiffies;
+ spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
+
+#ifdef CONFIG_SMP
+ if (pTimer->fSpecificCpu || pTimer->fAllCpus)
+ {
+# if RTLNX_VER_MIN(4,8,0)
+ mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
+# else
+ mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
+# endif
+ }
+ else
+#endif
+ mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
+ return;
+ }
+ break;
+
+ default:
+ AssertMsgFailed(("%d\n", enmState));
+ return;
+ }
+ ASMNopPause();
+ }
+}
+
+
+#ifdef CONFIG_SMP
+
+/**
+ * Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
+ *
+ * @param idCpu The current CPU.
+ * @param pvUser1 Pointer to the timer.
+ * @param pvUser2 Pointer to the argument structure.
+ */
+static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+ PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
+ PRTTIMER pTimer = (PRTTIMER)pvUser1;
+ Assert(idCpu < pTimer->cCpus);
+ rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes);
+}
+
+
+/**
+ * Worker for RTTimerStart() that takes care of the ugly bits.
+ *
+ * @returns RTTimerStart() return value.
+ * @param pTimer The timer.
+ * @param pArgs The argument structure.
+ */
+static int rtTimerLnxOmniStart(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
+{
+ RTCPUID iCpu;
+ RTCPUSET OnlineSet;
+ RTCPUSET OnlineSet2;
+ int rc2;
+
+ /*
+ * Prepare all the sub-timers for the startup and then flag the timer
+ * as a whole as non-suspended, make sure we get them all before
+ * clearing fSuspended as the MP handler will be waiting on this
+ * should something happen while we're looping.
+ */
+ RTSpinlockAcquire(pTimer->hSpinlock);
+
+ /* Just make it a omni timer restriction that no stop/start races are allowed. */
+ for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
+ if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
+ {
+ RTSpinlockRelease(pTimer->hSpinlock);
+ return VERR_TIMER_BUSY;
+ }
+
+ do
+ {
+ RTMpGetOnlineSet(&OnlineSet);
+ for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
+ {
+ Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
+ rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
+ RTCpuSetIsMember(&OnlineSet, iCpu)
+ ? RTTIMERLNXSTATE_STARTING
+ : RTTIMERLNXSTATE_STOPPED);
+ }
+ } while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
+
+ ASMAtomicWriteBool(&pTimer->fSuspended, false);
+
+ RTSpinlockRelease(pTimer->hSpinlock);
+
+ /*
+ * Start them (can't find any exported function that allows me to
+ * do this without the cross calls).
+ */
+ pArgs->u64Now = RTTimeSystemNanoTS();
+ rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
+ AssertRC(rc2); /* screw this if it fails. */
+
+ /*
+ * Reset the sub-timers who didn't start up (ALL CPUs case).
+ */
+ RTSpinlockAcquire(pTimer->hSpinlock);
+
+ for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
+ if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
+ {
+ /** @todo very odd case for a rainy day. Cpus that temporarily went offline while
+ * we were between calls needs to nudged as the MP handler will ignore events for
+ * them because of the STARTING state. This is an extremely unlikely case - not that
+ * that means anything in my experience... ;-) */
+ RTTIMERLNX_LOG(("what!? iCpu=%u -> didn't start\n", iCpu));
+ }
+
+ RTSpinlockRelease(pTimer->hSpinlock);
+
+ return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for RTTimerStop() that takes care of the ugly SMP bits.
+ *
+ * @returns true if there was any active callbacks, false if not.
+ * @param pTimer The timer (valid).
+ * @param fForDestroy Whether this is for RTTimerDestroy or not.
+ */
+static bool rtTimerLnxOmniStop(PRTTIMER pTimer, bool fForDestroy)
+{
+ bool fActiveCallbacks = false;
+ RTCPUID iCpu;
+ RTTIMERLNXSTATE enmState;
+
+
+ /*
+ * Mark the timer as suspended and flag all timers as stopping, except
+ * for those being stopped by an MP event.
+ */
+ RTSpinlockAcquire(pTimer->hSpinlock);
+
+ ASMAtomicWriteBool(&pTimer->fSuspended, true);
+ for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
+ {
+ for (;;)
+ {
+ enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
+ if ( enmState == RTTIMERLNXSTATE_STOPPED
+ || enmState == RTTIMERLNXSTATE_MP_STOPPING)
+ break;
+ if ( enmState == RTTIMERLNXSTATE_CALLBACK
+ || enmState == RTTIMERLNXSTATE_CB_STOPPING
+ || enmState == RTTIMERLNXSTATE_CB_RESTARTING)
+ {
+ Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING || fForDestroy);
+ if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState,
+ !fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
+ enmState))
+ {
+ fActiveCallbacks = true;
+ break;
+ }
+ }
+ else
+ {
+ Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
+ if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState))
+ break;
+ }
+ ASMNopPause();
+ }
+ }
+
+ RTSpinlockRelease(pTimer->hSpinlock);
+
+ /*
+ * Do the actual stopping. Fortunately, this doesn't require any IPIs.
+ * Unfortunately it cannot be done synchronously.
+ */
+ for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
+ if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
+ rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu], pTimer->fHighRes);
+
+ return fActiveCallbacks;
+}
+
+
+/**
+ * Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
+ * to start a sub-timer on a cpu that just have come online.
+ *
+ * @param idCpu The current CPU.
+ * @param pvUser1 Pointer to the timer.
+ * @param pvUser2 Pointer to the argument structure.
+ */
+static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+ PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
+ PRTTIMER pTimer = (PRTTIMER)pvUser1;
+ RTSPINLOCK hSpinlock;
+ Assert(idCpu < pTimer->cCpus);
+
+ /*
+ * We have to be kind of careful here as we might be racing RTTimerStop
+ * (and/or RTTimerDestroy, thus the paranoia.
+ */
+ hSpinlock = pTimer->hSpinlock;
+ if ( hSpinlock != NIL_RTSPINLOCK
+ && pTimer->u32Magic == RTTIMER_MAGIC)
+ {
+ RTSpinlockAcquire(hSpinlock);
+
+ if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
+ && pTimer->u32Magic == RTTIMER_MAGIC)
+ {
+ /* We're sane and the timer is not suspended yet. */
+ PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
+ rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes);
+ }
+
+ RTSpinlockRelease(hSpinlock);
+ }
+}
+
+
+/**
+ * MP event notification callback.
+ *
+ * @param enmEvent The event.
+ * @param idCpu The cpu it applies to.
+ * @param pvUser The timer.
+ */
+static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
+{
+ PRTTIMER pTimer = (PRTTIMER)pvUser;
+ PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
+ RTSPINLOCK hSpinlock;
+
+ Assert(idCpu < pTimer->cCpus);
+
+ /*
+ * Some initial paranoia.
+ */
+ if (pTimer->u32Magic != RTTIMER_MAGIC)
+ return;
+ hSpinlock = pTimer->hSpinlock;
+ if (hSpinlock == NIL_RTSPINLOCK)
+ return;
+
+ RTSpinlockAcquire(hSpinlock);
+
+ /* Is it active? */
+ if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
+ && pTimer->u32Magic == RTTIMER_MAGIC)
+ {
+ switch (enmEvent)
+ {
+ /*
+ * Try do it without leaving the spin lock, but if we have to, retake it
+ * when we're on the right cpu.
+ */
+ case RTMPEVENT_ONLINE:
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
+ {
+ RTTIMERLINUXSTARTONCPUARGS Args;
+ Args.u64Now = RTTimeSystemNanoTS();
+ Args.u64First = 0;
+
+ if (RTMpCpuId() == idCpu)
+ rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /*fPinned*/, pTimer->fHighRes);
+ else
+ {
+ rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
+ RTSpinlockRelease(hSpinlock);
+
+ RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
+ return; /* we've left the spinlock */
+ }
+ }
+ break;
+
+ /*
+ * The CPU is (going) offline, make sure the sub-timer is stopped.
+ *
+ * Linux will migrate it to a different CPU, but we don't want this. The
+ * timer function is checking for this.
+ */
+ case RTMPEVENT_OFFLINE:
+ {
+ RTTIMERLNXSTATE enmState;
+ while ( (enmState = rtTimerLnxGetState(&pSubTimer->enmState)) == RTTIMERLNXSTATE_ACTIVE
+ || enmState == RTTIMERLNXSTATE_CALLBACK
+ || enmState == RTTIMERLNXSTATE_CB_RESTARTING)
+ {
+ if (enmState == RTTIMERLNXSTATE_ACTIVE)
+ {
+ if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
+ {
+ RTSpinlockRelease(hSpinlock);
+
+ rtTimerLnxStopSubTimer(pSubTimer, pTimer->fHighRes);
+ return; /* we've left the spinlock */
+ }
+ }
+ else if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CB_STOPPING, enmState))
+ break;
+
+ /* State not stable, try again. */
+ ASMNopPause();
+ }
+ break;
+ }
+ }
+ }
+
+ RTSpinlockRelease(hSpinlock);
+}
+
+#endif /* CONFIG_SMP */
+
+
+/**
+ * Callback function use by RTTimerStart via RTMpOnSpecific to start a timer
+ * running on a specific CPU.
+ *
+ * @param idCpu The current CPU.
+ * @param pvUser1 Pointer to the timer.
+ * @param pvUser2 Pointer to the argument structure.
+ */
+static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+ PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
+ PRTTIMER pTimer = (PRTTIMER)pvUser1;
+ RT_NOREF_PV(idCpu);
+ rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes);
+}
+
+
+RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
+{
+ RTTIMERLINUXSTARTONCPUARGS Args;
+ int rc2;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate.
+ */
+ AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
+ AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
+
+ if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
+ return VERR_TIMER_ACTIVE;
+ RTTIMERLNX_LOG(("start %p cCpus=%d\n", pTimer, pTimer->cCpus));
+
+ Args.u64First = u64First;
+#ifdef CONFIG_SMP
+ /*
+ * Omni timer?
+ */
+ if (pTimer->fAllCpus)
+ {
+ rc2 = rtTimerLnxOmniStart(pTimer, &Args);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc2;
+ }
+#endif
+
+ /*
+ * Simple timer - Pretty straight forward if it wasn't for restarting.
+ */
+ Args.u64Now = RTTimeSystemNanoTS();
+ ASMAtomicWriteU64(&pTimer->aSubTimers[0].uNsRestartAt, Args.u64Now + u64First);
+ for (;;)
+ {
+ RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
+ switch (enmState)
+ {
+ case RTTIMERLNXSTATE_STOPPED:
+ if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING, RTTIMERLNXSTATE_STOPPED))
+ {
+ ASMAtomicWriteBool(&pTimer->fSuspended, false);
+ if (!pTimer->fSpecificCpu)
+ rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First,
+ false /*fPinned*/, pTimer->fHighRes);
+ else
+ {
+ rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
+ if (RT_FAILURE(rc2))
+ {
+ /* Suspend it, the cpu id is probably invalid or offline. */
+ ASMAtomicWriteBool(&pTimer->fSuspended, true);
+ rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
+ return rc2;
+ }
+ }
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ break;
+
+ case RTTIMERLNXSTATE_CALLBACK:
+ case RTTIMERLNXSTATE_CB_STOPPING:
+ if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_CB_RESTARTING, enmState))
+ {
+ ASMAtomicWriteBool(&pTimer->fSuspended, false);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+ break;
+
+ default:
+ AssertMsgFailed(("%d\n", enmState));
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_INTERNAL_ERROR_4;
+ }
+ ASMNopPause();
+ }
+}
+RT_EXPORT_SYMBOL(RTTimerStart);
+
+
+/**
+ * Common worker for RTTimerStop and RTTimerDestroy.
+ *
+ * @returns true if there was any active callbacks, false if not.
+ * @param pTimer The timer to stop.
+ * @param fForDestroy Whether it's RTTimerDestroy calling or not.
+ */
+static bool rtTimerLnxStop(PRTTIMER pTimer, bool fForDestroy)
+{
+ RTTIMERLNX_LOG(("lnxstop %p %d\n", pTimer, fForDestroy));
+#ifdef CONFIG_SMP
+ /*
+ * Omni timer?
+ */
+ if (pTimer->fAllCpus)
+ return rtTimerLnxOmniStop(pTimer, fForDestroy);
+#endif
+
+ /*
+ * Simple timer.
+ */
+ ASMAtomicWriteBool(&pTimer->fSuspended, true);
+ for (;;)
+ {
+ RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
+ switch (enmState)
+ {
+ case RTTIMERLNXSTATE_ACTIVE:
+ if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING, RTTIMERLNXSTATE_ACTIVE))
+ {
+ rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0], pTimer->fHighRes);
+ return false;
+ }
+ break;
+
+ case RTTIMERLNXSTATE_CALLBACK:
+ case RTTIMERLNXSTATE_CB_RESTARTING:
+ case RTTIMERLNXSTATE_CB_STOPPING:
+ Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING || fForDestroy);
+ if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState,
+ !fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
+ enmState))
+ return true;
+ break;
+
+ case RTTIMERLNXSTATE_STOPPED:
+ return VINF_SUCCESS;
+
+ case RTTIMERLNXSTATE_CB_DESTROYING:
+ AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
+ return true;
+
+ default:
+ case RTTIMERLNXSTATE_STARTING:
+ case RTTIMERLNXSTATE_MP_STARTING:
+ case RTTIMERLNXSTATE_STOPPING:
+ case RTTIMERLNXSTATE_MP_STOPPING:
+ AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
+ return false;
+ }
+
+ /* State not stable, try again. */
+ ASMNopPause();
+ }
+}
+
+
+RTDECL(int) RTTimerStop(PRTTIMER pTimer)
+{
+ /*
+ * Validate.
+ */
+ IPRT_LINUX_SAVE_EFL_AC();
+ AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
+ AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
+ RTTIMERLNX_LOG(("stop %p\n", pTimer));
+
+ if (ASMAtomicUoReadBool(&pTimer->fSuspended))
+ return VERR_TIMER_SUSPENDED;
+
+ rtTimerLnxStop(pTimer, false /*fForDestroy*/);
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTTimerStop);
+
+
+RTDECL(int) RTTimerChangeInterval(PRTTIMER pTimer, uint64_t u64NanoInterval)
+{
+ unsigned long cJiffies;
+ unsigned long flFlags;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate.
+ */
+ AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
+ AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
+ AssertReturn(u64NanoInterval, VERR_INVALID_PARAMETER);
+ AssertReturn(u64NanoInterval < UINT64_MAX / 8, VERR_INVALID_PARAMETER);
+ AssertReturn(pTimer->u64NanoInterval, VERR_INVALID_STATE);
+ RTTIMERLNX_LOG(("change %p %llu\n", pTimer, u64NanoInterval));
+
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+ /*
+ * For the high resolution timers it is easy since we don't care so much
+ * about when it is applied to the sub-timers.
+ */
+ if (pTimer->fHighRes)
+ {
+ ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+ }
+#endif
+
+ /*
+ * Standard timers have a bit more complicated way of calculating
+ * their interval and such. So, forget omni timers for now.
+ */
+ if (pTimer->cCpus > 1)
+ return VERR_NOT_SUPPORTED;
+
+ cJiffies = u64NanoInterval / (RT_NS_1SEC / HZ);
+ if (cJiffies * (RT_NS_1SEC / HZ) != u64NanoInterval)
+ cJiffies = 0;
+
+ spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
+ pTimer->aSubTimers[0].u.Std.fFirstAfterChg = true;
+ pTimer->cJiffies = cJiffies;
+ ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
+ spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTTimerChangeInterval);
+
+
+RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
+{
+ bool fCanDestroy;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ /*
+ * Validate. It's ok to pass NULL pointer.
+ */
+ if (pTimer == /*NIL_RTTIMER*/ NULL)
+ return VINF_SUCCESS;
+ AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
+ AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
+ RTTIMERLNX_LOG(("destroy %p\n", pTimer));
+/** @todo We should invalidate the magic here! */
+
+ /*
+ * Stop the timer if it's still active, then destroy it if we can.
+ */
+ if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
+ fCanDestroy = rtTimerLnxStop(pTimer, true /*fForDestroy*/);
+ else
+ {
+ uint32_t iCpu = pTimer->cCpus;
+ if (pTimer->cCpus > 1)
+ RTSpinlockAcquire(pTimer->hSpinlock);
+
+ fCanDestroy = true;
+ while (iCpu-- > 0)
+ {
+ for (;;)
+ {
+ RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
+ switch (enmState)
+ {
+ case RTTIMERLNXSTATE_CALLBACK:
+ case RTTIMERLNXSTATE_CB_RESTARTING:
+ case RTTIMERLNXSTATE_CB_STOPPING:
+ if (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_CB_DESTROYING, enmState))
+ continue;
+ fCanDestroy = false;
+ break;
+
+ case RTTIMERLNXSTATE_CB_DESTROYING:
+ AssertMsgFailed(("%d\n", enmState));
+ fCanDestroy = false;
+ break;
+ default:
+ break;
+ }
+ break;
+ }
+ }
+
+ if (pTimer->cCpus > 1)
+ RTSpinlockRelease(pTimer->hSpinlock);
+ }
+
+ if (fCanDestroy)
+ {
+ /* For paranoid reasons, defer actually destroying the semaphore when
+ in atomic or interrupt context. */
+#if RTLNX_VER_MIN(2,5,32)
+ if (in_atomic() || in_interrupt())
+#else
+ if (in_interrupt())
+#endif
+ rtR0LnxWorkqueuePush(&pTimer->DtorWorkqueueItem, rtTimerLnxDestroyDeferred);
+ else
+ rtTimerLnxDestroyIt(pTimer);
+ }
+
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTTimerDestroy);
+
+
+RTDECL(int) RTTimerCreateEx(PRTTIMER *ppTimer, uint64_t u64NanoInterval, uint32_t fFlags, PFNRTTIMER pfnTimer, void *pvUser)
+{
+ PRTTIMER pTimer;
+ RTCPUID iCpu;
+ unsigned cCpus;
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC();
+
+ rtR0LnxWorkqueueFlush(); /* for 2.4 */
+ *ppTimer = NULL;
+
+ /*
+ * Validate flags.
+ */
+ if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_INVALID_PARAMETER;
+ }
+ if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
+ && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
+ && !RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)))
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VERR_CPU_NOT_FOUND;
+ }
+
+ /*
+ * Allocate the timer handler.
+ */
+ cCpus = 1;
+#ifdef CONFIG_SMP
+ if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
+ {
+ cCpus = RTMpGetMaxCpuId() + 1;
+ Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
+ AssertReturnStmt(u64NanoInterval, IPRT_LINUX_RESTORE_EFL_AC(), VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
+ }
+#endif
+
+ rc = RTMemAllocEx(RT_UOFFSETOF_DYN(RTTIMER, aSubTimers[cCpus]), 0,
+ RTMEMALLOCEX_FLAGS_ZEROED | RTMEMALLOCEX_FLAGS_ANY_CTX_FREE, (void **)&pTimer);
+ if (RT_FAILURE(rc))
+ {
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+ }
+
+ /*
+ * Initialize it.
+ */
+ pTimer->u32Magic = RTTIMER_MAGIC;
+ pTimer->hSpinlock = NIL_RTSPINLOCK;
+ pTimer->fSuspended = true;
+ pTimer->fHighRes = !!(fFlags & RTTIMER_FLAGS_HIGH_RES);
+#ifdef CONFIG_SMP
+ pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
+ pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
+ pTimer->idCpu = pTimer->fSpecificCpu
+ ? RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)
+ : NIL_RTCPUID;
+#else
+ pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
+ pTimer->idCpu = RTMpCpuId();
+#endif
+ pTimer->cCpus = cCpus;
+ pTimer->pfnTimer = pfnTimer;
+ pTimer->pvUser = pvUser;
+ pTimer->u64NanoInterval = u64NanoInterval;
+ pTimer->cJiffies = u64NanoInterval / (RT_NS_1SEC / HZ);
+ if (pTimer->cJiffies * (RT_NS_1SEC / HZ) != u64NanoInterval)
+ pTimer->cJiffies = 0;
+ spin_lock_init(&pTimer->ChgIntLock);
+
+ for (iCpu = 0; iCpu < cCpus; iCpu++)
+ {
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+ if (pTimer->fHighRes)
+ {
+ hrtimer_init(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+ pTimer->aSubTimers[iCpu].u.Hr.LnxTimer.function = rtTimerLinuxHrCallback;
+ }
+ else
+#endif
+ {
+#if RTLNX_VER_MIN(4,15,0)
+ timer_setup(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer, rtTimerLinuxStdCallback, TIMER_PINNED);
+#elif RTLNX_VER_MIN(4,8,0)
+ init_timer_pinned(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
+#else
+ init_timer(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
+#endif
+#if RTLNX_VER_MAX(4,15,0)
+ pTimer->aSubTimers[iCpu].u.Std.LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
+ pTimer->aSubTimers[iCpu].u.Std.LnxTimer.function = rtTimerLinuxStdCallback;
+#endif
+ pTimer->aSubTimers[iCpu].u.Std.LnxTimer.expires = jiffies;
+ pTimer->aSubTimers[iCpu].u.Std.u64NextTS = 0;
+ }
+ pTimer->aSubTimers[iCpu].iTick = 0;
+ pTimer->aSubTimers[iCpu].pParent = pTimer;
+ pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
+ }
+
+#ifdef CONFIG_SMP
+ /*
+ * If this is running on ALL cpus, we'll have to register a callback
+ * for MP events (so timers can be started/stopped on cpus going
+ * online/offline). We also create the spinlock for synchronizing
+ * stop/start/mp-event.
+ */
+ if (cCpus > 1)
+ {
+ int rc = RTSpinlockCreate(&pTimer->hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "RTTimerLnx");
+ if (RT_SUCCESS(rc))
+ rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
+ else
+ pTimer->hSpinlock = NIL_RTSPINLOCK;
+ if (RT_FAILURE(rc))
+ {
+ RTTimerDestroy(pTimer);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return rc;
+ }
+ }
+#endif /* CONFIG_SMP */
+
+ RTTIMERLNX_LOG(("create %p hires=%d fFlags=%#x cCpus=%u\n", pTimer, pTimer->fHighRes, fFlags, cCpus));
+ *ppTimer = pTimer;
+ IPRT_LINUX_RESTORE_EFL_AC();
+ return VINF_SUCCESS;
+}
+RT_EXPORT_SYMBOL(RTTimerCreateEx);
+
+
+RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
+{
+#if 0 /** @todo Not sure if this is what we want or not... Add new API for
+ * querying the resolution of the high res timers? */
+ struct timespec Ts;
+ int rc;
+ IPRT_LINUX_SAVE_EFL_AC();
+ rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
+ IPRT_LINUX_RESTORE_EFL_AC();
+ if (!rc)
+ {
+ Assert(!Ts.tv_sec);
+ return Ts.tv_nsec;
+ }
+#endif
+ /* */
+#if RTLNX_VER_MAX(4,9,0) || RTLNX_VER_MIN(4,13,0)
+ /* On 4.9, 4.10 and 4.12 we've observed tstRTR0Timer failures of the omni timer tests
+ where we get about half of the ticks we want. The failing test is using this value
+ as interval. So, this is a very very crude hack to try make omni timers work
+ correctly without actually knowing what's going wrong... */
+ return RT_NS_1SEC * 2 / HZ; /* ns */
+#else
+ return RT_NS_1SEC / HZ; /* ns */
+#endif
+}
+RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
+
+
+RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
+{
+ RT_NOREF_PV(u32Request); RT_NOREF_PV(*pu32Granted);
+ return VERR_NOT_SUPPORTED;
+}
+RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
+
+
+RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
+{
+ RT_NOREF_PV(u32Granted);
+ return VERR_NOT_SUPPORTED;
+}
+RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
+
+
+RTDECL(bool) RTTimerCanDoHighResolution(void)
+{
+#ifdef RTTIMER_LINUX_WITH_HRTIMER
+ return true;
+#else
+ return false;
+#endif
+}
+RT_EXPORT_SYMBOL(RTTimerCanDoHighResolution);
+
diff --git a/src/VBox/Runtime/r0drv/linux/waitqueue-r0drv-linux.h b/src/VBox/Runtime/r0drv/linux/waitqueue-r0drv-linux.h
new file mode 100644
index 00000000..0ef103fe
--- /dev/null
+++ b/src/VBox/Runtime/r0drv/linux/waitqueue-r0drv-linux.h
@@ -0,0 +1,302 @@
+/* $Id: waitqueue-r0drv-linux.h $ */
+/** @file
+ * IPRT - Linux Ring-0 Driver Helpers for Abstracting Wait Queues,
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+ * in the VirtualBox distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+ */
+
+#ifndef IPRT_INCLUDED_SRC_r0drv_linux_waitqueue_r0drv_linux_h
+#define IPRT_INCLUDED_SRC_r0drv_linux_waitqueue_r0drv_linux_h
+#ifndef RT_WITHOUT_PRAGMA_ONCE
+# pragma once
+#endif
+
+#include "the-linux-kernel.h"
+
+#include <iprt/asm-math.h>
+#include <iprt/err.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+
+/** The resolution (nanoseconds) specified when using
+ * schedule_hrtimeout_range. */
+#define RTR0SEMLNXWAIT_RESOLUTION 50000
+
+
+/**
+ * Kernel mode Linux wait state structure.
+ */
+typedef struct RTR0SEMLNXWAIT
+{
+ /** The wait queue entry. */
+#if RTLNX_VER_MIN(4,13,0) || RTLNX_SUSE_MAJ_PREREQ(12, 4) || RTLNX_SUSE_MAJ_PREREQ(15, 0)
+ wait_queue_entry_t WaitQE;
+#else
+ wait_queue_t WaitQE;
+#endif
+ /** The absolute timeout given as nano seconds since the start of the
+ * monotonic clock. */
+ uint64_t uNsAbsTimeout;
+ /** The timeout in nano seconds relative to the start of the wait. */
+ uint64_t cNsRelTimeout;
+ /** The native timeout value. */
+ union
+ {
+#ifdef IPRT_LINUX_HAS_HRTIMER
+ /** The timeout when fHighRes is true. Absolute, so no updating. */
+ ktime_t KtTimeout;
+#endif
+ /** The timeout when fHighRes is false. Updated after waiting. */
+ long lTimeout;
+ } u;
+ /** Set if we use high resolution timeouts. */
+ bool fHighRes;
+ /** Set if it's an indefinite wait. */
+ bool fIndefinite;
+ /** Set if we've already timed out.
+ * Set by rtR0SemLnxWaitDoIt and read by rtR0SemLnxWaitHasTimedOut. */
+ bool fTimedOut;
+ /** TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE. */
+ int iWaitState;
+ /** The wait queue. */
+ wait_queue_head_t *pWaitQueue;
+} RTR0SEMLNXWAIT;
+/** Pointer to a linux wait state. */
+typedef RTR0SEMLNXWAIT *PRTR0SEMLNXWAIT;
+
+
+/**
+ * Initializes a wait.
+ *
+ * The caller MUST check the wait condition BEFORE calling this function or the
+ * timeout logic will be flawed.
+ *
+ * @returns VINF_SUCCESS or VERR_TIMEOUT.
+ * @param pWait The wait structure.
+ * @param fFlags The wait flags.
+ * @param uTimeout The timeout.
+ * @param pWaitQueue The wait queue head.
+ */
+DECLINLINE(int) rtR0SemLnxWaitInit(PRTR0SEMLNXWAIT pWait, uint32_t fFlags, uint64_t uTimeout,
+ wait_queue_head_t *pWaitQueue)
+{
+ /*
+ * Process the flags and timeout.
+ */
+ if (!(fFlags & RTSEMWAIT_FLAGS_INDEFINITE))
+ {
+/** @todo optimize: millisecs -> nanosecs -> millisec -> jiffies */
+ if (fFlags & RTSEMWAIT_FLAGS_MILLISECS)
+ uTimeout = uTimeout < UINT64_MAX / RT_US_1SEC * RT_US_1SEC
+ ? uTimeout * RT_US_1SEC
+ : UINT64_MAX;
+ if (uTimeout == UINT64_MAX)
+ fFlags |= RTSEMWAIT_FLAGS_INDEFINITE;
+ else
+ {
+ uint64_t u64Now;
+ if (fFlags & RTSEMWAIT_FLAGS_RELATIVE)
+ {
+ if (uTimeout == 0)
+ return VERR_TIMEOUT;
+
+ u64Now = RTTimeSystemNanoTS();
+ pWait->cNsRelTimeout = uTimeout;
+ pWait->uNsAbsTimeout = u64Now + uTimeout;
+ if (pWait->uNsAbsTimeout < u64Now) /* overflow */
+ fFlags |= RTSEMWAIT_FLAGS_INDEFINITE;
+ }
+ else
+ {
+ u64Now = RTTimeSystemNanoTS();
+ if (u64Now >= uTimeout)
+ return VERR_TIMEOUT;
+
+ pWait->cNsRelTimeout = uTimeout - u64Now;
+ pWait->uNsAbsTimeout = uTimeout;
+ }
+ }
+ }
+
+ if (!(fFlags & RTSEMWAIT_FLAGS_INDEFINITE))
+ {
+ pWait->fIndefinite = false;
+#ifdef IPRT_LINUX_HAS_HRTIMER
+ if ( (fFlags & (RTSEMWAIT_FLAGS_NANOSECS | RTSEMWAIT_FLAGS_ABSOLUTE))
+ || pWait->cNsRelTimeout < RT_NS_1SEC / HZ * 4)
+ {
+ pWait->fHighRes = true;
+# if BITS_PER_LONG < 64
+ if ( KTIME_SEC_MAX <= LONG_MAX
+ && pWait->uNsAbsTimeout >= KTIME_SEC_MAX * RT_NS_1SEC_64 + (RT_NS_1SEC - 1))
+ fFlags |= RTSEMWAIT_FLAGS_INDEFINITE;
+ else
+# endif
+ pWait->u.KtTimeout = ns_to_ktime(pWait->uNsAbsTimeout);
+ }
+ else
+#endif
+ {
+ uint64_t cJiffies = ASMMultU64ByU32DivByU32(pWait->cNsRelTimeout, HZ, RT_NS_1SEC);
+ if (cJiffies >= MAX_JIFFY_OFFSET)
+ fFlags |= RTSEMWAIT_FLAGS_INDEFINITE;
+ else
+ {
+ pWait->u.lTimeout = (long)cJiffies;
+ pWait->fHighRes = false;
+ }
+ }
+ }
+
+ if (fFlags & RTSEMWAIT_FLAGS_INDEFINITE)
+ {
+ pWait->fIndefinite = true;
+ pWait->fHighRes = false;
+ pWait->uNsAbsTimeout = UINT64_MAX;
+ pWait->cNsRelTimeout = UINT64_MAX;
+ pWait->u.lTimeout = LONG_MAX;
+ }
+
+ pWait->fTimedOut = false;
+
+ /*
+ * Initialize the wait queue related bits.
+ */
+#if RTLNX_VER_MIN(2,5,39)
+ init_wait((&pWait->WaitQE));
+#else
+ RT_ZERO(pWait->WaitQE);
+ init_waitqueue_entry((&pWait->WaitQE), current);
+#endif
+ pWait->pWaitQueue = pWaitQueue;
+ pWait->iWaitState = fFlags & RTSEMWAIT_FLAGS_INTERRUPTIBLE
+ ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
+
+ return VINF_SUCCESS;
+}
+
+
+/**
+ * Prepares the next wait.
+ *
+ * This must be called before rtR0SemLnxWaitDoIt, and the caller should check
+ * the exit conditions in-between the two calls.
+ *
+ * @param pWait The wait structure.
+ */
+DECLINLINE(void) rtR0SemLnxWaitPrepare(PRTR0SEMLNXWAIT pWait)
+{
+ /* Make everything thru schedule*() atomic scheduling wise. (Is this correct?) */
+ prepare_to_wait(pWait->pWaitQueue, &pWait->WaitQE, pWait->iWaitState);
+}
+
+
+/**
+ * Do the actual wait.
+ *
+ * @param pWait The wait structure.
+ */
+DECLINLINE(void) rtR0SemLnxWaitDoIt(PRTR0SEMLNXWAIT pWait)
+{
+ if (pWait->fIndefinite)
+ schedule();
+#ifdef IPRT_LINUX_HAS_HRTIMER
+ else if (pWait->fHighRes)
+ {
+ int rc = schedule_hrtimeout_range(&pWait->u.KtTimeout, HRTIMER_MODE_ABS, RTR0SEMLNXWAIT_RESOLUTION);
+ if (!rc)
+ pWait->fTimedOut = true;
+ }
+#endif
+ else
+ {
+ pWait->u.lTimeout = schedule_timeout(pWait->u.lTimeout);
+ if (pWait->u.lTimeout <= 0)
+ pWait->fTimedOut = true;
+ }
+ after_wait((&pWait->WaitQE));
+}
+
+
+/**
+ * Checks if a linux wait was interrupted.
+ *
+ * @returns true / false
+ * @param pWait The wait structure.
+ * @remarks This shall be called before the first rtR0SemLnxWaitDoIt().
+ */
+DECLINLINE(bool) rtR0SemLnxWaitWasInterrupted(PRTR0SEMLNXWAIT pWait)
+{
+ return pWait->iWaitState == TASK_INTERRUPTIBLE
+ && signal_pending(current);
+}
+
+
+/**
+ * Checks if a linux wait has timed out.
+ *
+ * @returns true / false
+ * @param pWait The wait structure.
+ */
+DECLINLINE(bool) rtR0SemLnxWaitHasTimedOut(PRTR0SEMLNXWAIT pWait)
+{
+ return pWait->fTimedOut;
+}
+
+
+/**
+ * Deletes a linux wait.
+ *
+ * @param pWait The wait structure.
+ */
+DECLINLINE(void) rtR0SemLnxWaitDelete(PRTR0SEMLNXWAIT pWait)
+{
+ finish_wait(pWait->pWaitQueue, &pWait->WaitQE);
+}
+
+
+/**
+ * Gets the max resolution of the timeout machinery.
+ *
+ * @returns Resolution specified in nanoseconds.
+ */
+DECLINLINE(uint32_t) rtR0SemLnxWaitGetResolution(void)
+{
+#ifdef IPRT_LINUX_HAS_HRTIMER
+ return RTR0SEMLNXWAIT_RESOLUTION;
+#else
+ return RT_NS_1SEC / HZ; /* ns */
+#endif
+}
+
+#endif /* !IPRT_INCLUDED_SRC_r0drv_linux_waitqueue_r0drv_linux_h */
+