path: root/src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp

/* $Id: memobj-r0drv-darwin.cpp $ */
/** @file
 * IPRT - Ring-0 Memory Objects, Darwin.
 */

/*
 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
 *
 * This file is part of VirtualBox base platform packages, as
 * available from https://www.virtualbox.org.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation, in version 3 of the
 * License.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <https://www.gnu.org/licenses>.
 *
 * The contents of this file may alternatively be used under the terms
 * of the Common Development and Distribution License Version 1.0
 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
 * in the VirtualBox distribution, in which case the provisions of the
 * CDDL are applicable instead of those of the GPL.
 *
 * You may elect to license modified versions of this file under the
 * terms and conditions of either the GPL or the CDDL or both.
 *
 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
 */


/*********************************************************************************************************************************
*   Header Files                                                                                                                 *
*********************************************************************************************************************************/
#define RTMEM_NO_WRAP_TO_EF_APIS /* circular dependency otherwise. */
#include "the-darwin-kernel.h"
#include "internal/iprt.h"
#include <iprt/memobj.h>

#include <iprt/asm.h>
#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
# include <iprt/x86.h>
# include <iprt/asm-amd64-x86.h>
#endif
#include <iprt/assert.h>
#include <iprt/log.h>
#include <iprt/mem.h>
#include <iprt/param.h>
#include <iprt/process.h>
#include <iprt/semaphore.h>
#include <iprt/string.h>
#include <iprt/thread.h>
#include "internal/memobj.h"


/*********************************************************************************************************************************
*   Defined Constants And Macros                                                                                                 *
*********************************************************************************************************************************/
#define MY_PRINTF(...) do { printf(__VA_ARGS__); kprintf(__VA_ARGS__); } while (0)

/*#define USE_VM_MAP_WIRE - may re-enable later when non-mapped allocations are added. */


/*********************************************************************************************************************************
*   Structures and Typedefs                                                                                                      *
*********************************************************************************************************************************/
/**
 * The Darwin version of the memory object structure.
 */
typedef struct RTR0MEMOBJDARWIN
{
    /** The core structure. */
    RTR0MEMOBJINTERNAL  Core;
    /** Pointer to the memory descriptor created for allocated and locked memory. */
    IOMemoryDescriptor *pMemDesc;
    /** Pointer to the memory mapping object for mapped memory. */
    IOMemoryMap        *pMemMap;
} RTR0MEMOBJDARWIN, *PRTR0MEMOBJDARWIN;

/**
 * Common thread_call_allocate/thread_call_enter argument package.
 */
typedef struct RTR0MEMOBJDARWINTHREADARGS
{
    int32_t volatile        rc;
    RTSEMEVENTMULTI         hEvent;
} RTR0MEMOBJDARWINTHREADARGS;


/**
 * Arguments for rtR0MemObjNativeAllockWorkOnKernelThread.
 */
typedef struct RTR0MEMOBJDARWINALLOCARGS
{
    RTR0MEMOBJDARWINTHREADARGS Core;
    PPRTR0MEMOBJINTERNAL    ppMem;
    size_t                  cb;
    bool                    fExecutable;
    bool                    fContiguous;
    mach_vm_address_t       PhysMask;
    uint64_t                MaxPhysAddr;
    RTR0MEMOBJTYPE          enmType;
    size_t                  uAlignment;
    const char             *pszTag;
} RTR0MEMOBJDARWINALLOCARGS;

/**
 * Arguments for rtR0MemObjNativeProtectWorkOnKernelThread.
 */
typedef struct RTR0MEMOBJDARWINPROTECTARGS
{
    RTR0MEMOBJDARWINTHREADARGS Core;
    PRTR0MEMOBJINTERNAL     pMem;
    size_t                  offSub;
    size_t                  cbSub;
    uint32_t                fProt;
} RTR0MEMOBJDARWINPROTECTARGS;


/*********************************************************************************************************************************
*   Internal Functions                                                                                                           *
*********************************************************************************************************************************/
static void rtR0MemObjNativeAllockWorkerOnKernelThread(void *pvUser0, void *pvUser1);
static int  rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt);
static void rtR0MemObjNativeProtectWorkerOnKernelThread(void *pvUser0, void *pvUser1);


/**
 * Touch the pages to force the kernel to create or write-enable the page table
 * entries.
 *
 * This is necessary since the kernel gets upset if we take a page fault when
 * preemption is disabled and/or we own a simple lock (same thing).  It has no
 * problems with us disabling interrupts when taking the traps, weird stuff.
 *
 * (This is basically a way of invoking vm_fault on a range of pages.)
 *
 * @param  pv           Pointer to the first page.
 * @param  cb           The number of bytes.
 */
static void rtR0MemObjDarwinTouchPages(void *pv, size_t cb)
{
    uint32_t volatile  *pu32 = (uint32_t volatile *)pv;
    for (;;)
    {
        ASMAtomicCmpXchgU32(pu32, 0xdeadbeef, 0xdeadbeef);
        if (cb <= PAGE_SIZE)
            break;
        cb -= PAGE_SIZE;
        pu32 += PAGE_SIZE / sizeof(uint32_t);
    }
}


/**
 * Read (sniff) every page in the range to make sure there are some page tables
 * entries backing it.
 *
 * This is just to be sure vm_protect didn't remove stuff without re-adding it
 * if someone should try write-protect something.
 *
 * @param  pv           Pointer to the first page.
 * @param  cb           The number of bytes.
 */
static void rtR0MemObjDarwinSniffPages(void const *pv, size_t cb)
{
    uint32_t volatile  *pu32 = (uint32_t volatile *)pv;
    uint32_t volatile   u32Counter = 0;
    for (;;)
    {
        u32Counter += *pu32;

        if (cb <= PAGE_SIZE)
            break;
        cb -= PAGE_SIZE;
        pu32 += PAGE_SIZE / sizeof(uint32_t);
    }
}


/**
 * Gets the virtual memory map the specified object is mapped into.
 *
 * @returns VM map handle on success, NULL if no map.
 * @param   pMem                The memory object.
 */
DECLINLINE(vm_map_t) rtR0MemObjDarwinGetMap(PRTR0MEMOBJINTERNAL pMem)
{
    switch (pMem->enmType)
    {
        case RTR0MEMOBJTYPE_PAGE:
        case RTR0MEMOBJTYPE_LOW:
        case RTR0MEMOBJTYPE_CONT:
            return kernel_map;

        case RTR0MEMOBJTYPE_PHYS:
        case RTR0MEMOBJTYPE_PHYS_NC:
            if (pMem->pv)
                return kernel_map;
            return NULL;

        case RTR0MEMOBJTYPE_LOCK:
            return pMem->u.Lock.R0Process == NIL_RTR0PROCESS
                ? kernel_map
                : get_task_map((task_t)pMem->u.Lock.R0Process);

        case RTR0MEMOBJTYPE_RES_VIRT:
            return pMem->u.ResVirt.R0Process == NIL_RTR0PROCESS
                ? kernel_map
                : get_task_map((task_t)pMem->u.ResVirt.R0Process);

        case RTR0MEMOBJTYPE_MAPPING:
            return pMem->u.Mapping.R0Process == NIL_RTR0PROCESS
                ? kernel_map
                : get_task_map((task_t)pMem->u.Mapping.R0Process);

        default:
            return NULL;
    }
}

#if 0 /* not necessary after all*/
/* My vm_map mockup. */
struct my_vm_map
{
    struct { char pad[8]; } lock;
    struct my_vm_map_header
    {
        struct vm_map_links
        {
            void            *prev;
            void            *next;
            vm_map_offset_t start;
            vm_map_offset_t end;
        }                   links;
        int                 nentries;
        boolean_t           entries_pageable;
    }                       hdr;
    pmap_t                  pmap;
    vm_map_size_t           size;
};


/**
 * Gets the minimum map address, this is similar to get_map_min.
 *
 * @returns The start address of the map.
 * @param   pMap                The map.
 */
static vm_map_offset_t rtR0MemObjDarwinGetMapMin(vm_map_t pMap)
{
    /* lazy discovery of the correct offset. The apple guys is a wonderfully secretive bunch. */
    static int32_t volatile s_offAdjust = INT32_MAX;
    int32_t                 off         = s_offAdjust;
    if (off == INT32_MAX)
    {
        for (off = 0; ; off += sizeof(pmap_t))
        {
            if (*(pmap_t *)((uint8_t *)kernel_map + off) == kernel_pmap)
                break;
            AssertReturn(off <= RT_MAX(RT_OFFSETOF(struct my_vm_map, pmap) * 4, 1024), 0x1000);
        }
        ASMAtomicWriteS32(&s_offAdjust, off - RT_OFFSETOF(struct my_vm_map, pmap));
    }

    /* calculate it. */
    struct my_vm_map *pMyMap = (struct my_vm_map *)((uint8_t *)pMap + off);
    return pMyMap->hdr.links.start;
}
#endif /* unused */

#ifdef RT_STRICT
# if 0 /* unused */

/**
 * Read from a physical page.
 *
 * @param   HCPhys      The address to start reading at.
 * @param   cb          How many bytes to read.
 * @param   pvDst       Where to put the bytes. This is zero'd on failure.
 */
static void rtR0MemObjDarwinReadPhys(RTHCPHYS HCPhys, size_t cb, void *pvDst)
{
    memset(pvDst, '\0', cb);

    IOAddressRange      aRanges[1]  = { { (mach_vm_address_t)HCPhys, RT_ALIGN_Z(cb, PAGE_SIZE) } };
    IOMemoryDescriptor *pMemDesc    = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges),
                                                                            kIODirectionIn, NULL /*task*/);
    if (pMemDesc)
    {
#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
        IOMemoryMap *pMemMap = pMemDesc->createMappingInTask(kernel_task, 0, kIOMapAnywhere | kIOMapDefaultCache);
#else
        IOMemoryMap *pMemMap = pMemDesc->map(kernel_task, 0, kIOMapAnywhere | kIOMapDefaultCache);
#endif
        if (pMemMap)
        {
            void const *pvSrc = (void const *)(uintptr_t)pMemMap->getVirtualAddress();
            memcpy(pvDst, pvSrc, cb);
            pMemMap->release();
        }
        else
            MY_PRINTF("rtR0MemObjDarwinReadPhys: createMappingInTask failed; HCPhys=%llx\n", HCPhys);

        pMemDesc->release();
    }
    else
        MY_PRINTF("rtR0MemObjDarwinReadPhys: withAddressRanges failed; HCPhys=%llx\n", HCPhys);
}


/**
 * Gets the PTE for a page.
 *
 * @returns the PTE.
 * @param   pvPage      The virtual address to get the PTE for.
 */
static uint64_t rtR0MemObjDarwinGetPTE(void *pvPage)
{
    RTUINT64U   u64;
    RTCCUINTREG cr3 = ASMGetCR3();
    RTCCUINTREG cr4 = ASMGetCR4();
    bool        fPAE = false;
    bool        fLMA = false;
    if (cr4 & X86_CR4_PAE)
    {
        fPAE = true;
        uint32_t fExtFeatures = ASMCpuId_EDX(0x80000001);
        if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
        {
            uint64_t efer = ASMRdMsr(MSR_K6_EFER);
            if (efer & MSR_K6_EFER_LMA)
                fLMA = true;
        }
    }

    if (fLMA)
    {
        /* PML4 */
        rtR0MemObjDarwinReadPhys((cr3 & ~(RTCCUINTREG)PAGE_OFFSET_MASK) | (((uint64_t)(uintptr_t)pvPage >> X86_PML4_SHIFT) & X86_PML4_MASK) * 8, 8, &u64);
        if (!(u64.u & X86_PML4E_P))
        {
            MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PML4E !p\n", pvPage);
            return 0;
        }

        /* PDPTR */
        rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64) * 8, 8, &u64);
        if (!(u64.u & X86_PDPE_P))
        {
            MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDPTE !p\n", pvPage);
            return 0;
        }
        if (u64.u & X86_PDPE_LM_PS)
            return (u64.u & ~(uint64_t)(_1G -1)) | ((uintptr_t)pvPage & (_1G -1));

        /* PD */
        rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64);
        if (!(u64.u & X86_PDE_P))
        {
            MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDE !p\n", pvPage);
            return 0;
        }
        if (u64.u & X86_PDE_PS)
            return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1));

        /* PT */
        rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64);
        if (!(u64.u &  X86_PTE_P))
        {
            MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PTE !p\n", pvPage);
            return 0;
        }
        return u64.u;
    }

    if (fPAE)
    {
        /* PDPTR */
        rtR0MemObjDarwinReadPhys((u64.u & X86_CR3_PAE_PAGE_MASK) | (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE) * 8, 8, &u64);
        if (!(u64.u & X86_PDE_P))
            return 0;

        /* PD */
        rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64);
        if (!(u64.u & X86_PDE_P))
            return 0;
        if (u64.u & X86_PDE_PS)
            return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1));

        /* PT */
        rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64);
        if (!(u64.u & X86_PTE_P))
            return 0;
        return u64.u;
    }

    /* PD */
    rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_SHIFT) & X86_PD_MASK) * 4, 4, &u64);
    if (!(u64.au32[0] & X86_PDE_P))
        return 0;
    if (u64.au32[0] & X86_PDE_PS)
        return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1));

    /* PT */
    rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_SHIFT) & X86_PT_MASK) * 4, 4, &u64);
    if (!(u64.au32[0] & X86_PTE_P))
        return 0;
    return u64.au32[0];

    return 0;
}

# endif /* unused */
#endif /* RT_STRICT */

DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
{
    PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)pMem;
    IPRT_DARWIN_SAVE_EFL_AC();

    /*
     * Release the IOMemoryDescriptor or/and IOMemoryMap associated with the object.
     */
    if (pMemDarwin->pMemDesc)
    {
        pMemDarwin->pMemDesc->complete();
        pMemDarwin->pMemDesc->release();
        pMemDarwin->pMemDesc = NULL;
    }

    if (pMemDarwin->pMemMap)
    {
        pMemDarwin->pMemMap->release();
        pMemDarwin->pMemMap = NULL;
    }

    /*
     * Release any memory that we've allocated or locked.
     */
    switch (pMemDarwin->Core.enmType)
    {
        case RTR0MEMOBJTYPE_LOW:
        case RTR0MEMOBJTYPE_PAGE:
        case RTR0MEMOBJTYPE_CONT:
            break;

        case RTR0MEMOBJTYPE_LOCK:
        {
#ifdef USE_VM_MAP_WIRE
            vm_map_t Map = pMemDarwin->Core.u.Lock.R0Process != NIL_RTR0PROCESS
                         ? get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process)
                         : kernel_map;
            kern_return_t kr = vm_map_unwire(Map,
                                             (vm_map_offset_t)pMemDarwin->Core.pv,
                                             (vm_map_offset_t)pMemDarwin->Core.pv + pMemDarwin->Core.cb,
                                             0 /* not user */);
            AssertRC(kr == KERN_SUCCESS); /** @todo don't ignore... */
#endif
            break;
        }

        case RTR0MEMOBJTYPE_PHYS:
            /*if (pMemDarwin->Core.u.Phys.fAllocated)
                IOFreePhysical(pMemDarwin->Core.u.Phys.PhysBase, pMemDarwin->Core.cb);*/
            Assert(!pMemDarwin->Core.u.Phys.fAllocated);
            break;

        case RTR0MEMOBJTYPE_PHYS_NC:
            AssertMsgFailed(("RTR0MEMOBJTYPE_PHYS_NC\n"));
            IPRT_DARWIN_RESTORE_EFL_AC();
            return VERR_INTERNAL_ERROR;

        case RTR0MEMOBJTYPE_RES_VIRT:
            AssertMsgFailed(("RTR0MEMOBJTYPE_RES_VIRT\n"));
            IPRT_DARWIN_RESTORE_EFL_AC();
            return VERR_INTERNAL_ERROR;

        case RTR0MEMOBJTYPE_MAPPING:
            /* nothing to do here. */
            break;

        default:
            AssertMsgFailed(("enmType=%d\n", pMemDarwin->Core.enmType));
            IPRT_DARWIN_RESTORE_EFL_AC();
            return VERR_INTERNAL_ERROR;
    }

    IPRT_DARWIN_RESTORE_EFL_AC();
    return VINF_SUCCESS;
}


/**
 * This is a helper function to executes @a pfnWorker in the context of the
 * kernel_task
 *
 * @returns IPRT status code - result from pfnWorker or dispatching error.
 * @param   pfnWorker       The function to call.
 * @param   pArgs           The arguments to pass to the function.
 */
static int rtR0MemObjDarwinDoInKernelTaskThread(thread_call_func_t pfnWorker, RTR0MEMOBJDARWINTHREADARGS *pArgs)
{
    pArgs->rc     = VERR_IPE_UNINITIALIZED_STATUS;
    pArgs->hEvent = NIL_RTSEMEVENTMULTI;
    int rc = RTSemEventMultiCreate(&pArgs->hEvent);
    if (RT_SUCCESS(rc))
    {
        thread_call_t hCall = thread_call_allocate(pfnWorker, (void *)pArgs);
        if (hCall)
        {
            boolean_t fRc = thread_call_enter(hCall);
            AssertLogRel(fRc == FALSE);

            rc = RTSemEventMultiWaitEx(pArgs->hEvent, RTSEMWAIT_FLAGS_INDEFINITE | RTSEMWAIT_FLAGS_UNINTERRUPTIBLE,
                                       RT_INDEFINITE_WAIT);
            AssertLogRelRC(rc);

            rc = pArgs->rc;
            thread_call_free(hCall);
        }
        else
            rc = VERR_NO_MEMORY;
        RTSemEventMultiDestroy(pArgs->hEvent);
    }
    return rc;
}


/**
 * Signals result to thread waiting in rtR0MemObjDarwinDoInKernelTaskThread.
 *
 * @param   pArgs           The argument structure.
 * @param   rc              The IPRT status code to signal.
 */
static void rtR0MemObjDarwinSignalThreadWaitinOnTask(RTR0MEMOBJDARWINTHREADARGS volatile *pArgs, int rc)
{
    if (ASMAtomicCmpXchgS32(&pArgs->rc, rc, VERR_IPE_UNINITIALIZED_STATUS))
    {
        rc = RTSemEventMultiSignal(pArgs->hEvent);
        AssertLogRelRC(rc);
    }
}


/**
 * Kernel memory alloc worker that uses inTaskWithPhysicalMask.
 *
 * @returns IPRT status code.
 * @retval  VERR_ADDRESS_TOO_BIG try another way.
 *
 * @param   ppMem           Where to return the memory object.
 * @param   cb              The page aligned memory size.
 * @param   fExecutable     Whether the mapping needs to be executable.
 * @param   fContiguous     Whether the backing memory needs to be contiguous.
 * @param   PhysMask        The mask for the backing memory (i.e. range). Use 0 if
 *                          you don't care that much or is speculating.
 * @param   MaxPhysAddr     The max address to verify the result against. Use
 *                          UINT64_MAX if it doesn't matter.
 * @param   enmType         The object type.
 * @param   uAlignment      The allocation alignment (in bytes).
 * @param   pszTag          Allocation tag used for statistics and such.
 * @param   fOnKernelThread Set if we're already on the kernel thread.
 */
static int rtR0MemObjNativeAllocWorker(PPRTR0MEMOBJINTERNAL ppMem, size_t cb,
                                       bool fExecutable, bool fContiguous,
                                       mach_vm_address_t PhysMask, uint64_t MaxPhysAddr,
                                       RTR0MEMOBJTYPE enmType, size_t uAlignment, const char *pszTag, bool fOnKernelThread)
{
    int rc;

    /*
     * Because of process code signing properties leaking into kernel space in
     * in XNU's vm_fault.c code, we have to defer allocations of exec memory to
     * a thread running in the kernel_task to get consistent results here.
     *
     * Trouble strikes in vm_fault_enter() when cs_enforcement_enabled is determined
     * to be true because current process has the CS_ENFORCEMENT flag, the page flag
     * vmp_cs_validated is clear, and the protection mask includes VM_PROT_EXECUTE
     * (pmap_cs_enforced does not apply to macOS it seems).  This test seems to go
     * back to 10.5, though I'm not sure whether it's enabled for macOS that early
     * on.  Only VM_PROT_EXECUTE is problematic for kernel memory, (though
     * VM_PROT_WRITE on code signed pages is also problematic in theory).  As long as
     * kernel_task doesn't have CS_ENFORCEMENT enabled, we'll be fine switching to it.
     */
    if (!fExecutable || fOnKernelThread)
    { /* likely */ }
    else
    {
        RTR0MEMOBJDARWINALLOCARGS Args;
        Args.ppMem          = ppMem;
        Args.cb             = cb;
        Args.fExecutable    = fExecutable;
        Args.fContiguous    = fContiguous;
        Args.PhysMask       = PhysMask;
        Args.MaxPhysAddr    = MaxPhysAddr;
        Args.enmType        = enmType;
        Args.uAlignment     = uAlignment;
        Args.pszTag         = pszTag;
        return rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeAllockWorkerOnKernelThread, &Args.Core);
    }

    /*
     * Try inTaskWithPhysicalMask first, but since we don't quite trust that it
     * actually respects the physical memory mask (10.5.x is certainly busted),
     * we'll use rtR0MemObjNativeAllocCont as a fallback for dealing with that.
     *
     * The kIOMemoryKernelUserShared flag just forces the result to be page aligned.
     *
     * The kIOMemoryMapperNone flag is required since 10.8.2 (IOMMU changes?).
     */

    /* This is an old fudge from the snow leoard days: "Is it only on snow leopard?
       Seen allocating memory for the VM structure, last page corrupted or
       inaccessible."  Made it only apply to snow leopard and older for now. */
    size_t cbFudged = cb;
    if (version_major >= 11 /* 10 = 10.7.x = Lion. */)
    { /* likely */ }
    else
         cbFudged += PAGE_SIZE;

    IOOptionBits fOptions = kIOMemoryKernelUserShared | kIODirectionInOut;
    if (fContiguous)
    {
        fOptions |= kIOMemoryPhysicallyContiguous;
        if (   version_major > 12
            || (version_major == 12 && version_minor >= 2) /* 10.8.2 = Mountain Kitten */ )
            fOptions |= kIOMemoryHostPhysicallyContiguous; /* (Just to make ourselves clear, in case the xnu code changes.)  */
    }
    if (version_major >= 12 /* 12 = 10.8.x = Mountain Kitten */)
        fOptions |= kIOMemoryMapperNone;

#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 && 0 /* enable when/if necessary */
    /* Paranoia: Don't misrepresent our intentions, we won't map kernel executable memory into ring-0. */
    if (fExecutable && version_major >= 11 /* 10.7.x = Lion, as below */)
    {
        fOptions &= ~kIOMemoryKernelUserShared;
        if (uAlignment < PAGE_SIZE)
            uAlignment = PAGE_SIZE;
    }
#endif

    /* The public initWithPhysicalMask virtual method appeared in 10.7.0, in
       versions 10.5.0 up to 10.7.0 it was private, and 10.4.8-10.5.0 it was
       x86 only and didn't have the alignment parameter (slot was different too). */
    uint64_t uAlignmentActual = uAlignment;
    IOBufferMemoryDescriptor *pMemDesc;
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070
    if (version_major >= 11 /* 11 = 10.7.x = Lion, could probably allow 10.5.0+ here if we really wanted to. */)
    {
        /* Starting with 10.6.x the physical mask is ignored if alignment is higher
           than 1.  The assumption seems to be that inTaskWithPhysicalMask() should
           be used and the alignment inferred from the PhysMask argument. */
        if (MaxPhysAddr != UINT64_MAX)
        {
            Assert(RT_ALIGN_64(PhysMask, uAlignment) == PhysMask);
            uAlignmentActual = 1;
        }

        pMemDesc = new IOBufferMemoryDescriptor;
        if (pMemDesc)
        {
            if (pMemDesc->initWithPhysicalMask(kernel_task, fOptions, cbFudged, uAlignmentActual, PhysMask))
            { /* likely */ }
            else
            {
                pMemDesc->release();
                pMemDesc = NULL;
            }
        }
    }
    else
#endif
        pMemDesc = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task, fOptions, cbFudged, PhysMask);
    if (pMemDesc)
    {
        IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
        if (IORet == kIOReturnSuccess)
        {
            void *pv = pMemDesc->getBytesNoCopy(0, cbFudged);
            if (pv)
            {
                /*
                 * Check if it's all below 4GB.
                 */
                addr64_t AddrPrev = 0;
                MaxPhysAddr &= ~(uint64_t)PAGE_OFFSET_MASK;
                for (IOByteCount off = 0; off < cb; off += PAGE_SIZE)
                {
#ifdef __LP64__
                    addr64_t Addr = pMemDesc->getPhysicalSegment(off, NULL, kIOMemoryMapperNone);
#else
                    addr64_t Addr = pMemDesc->getPhysicalSegment64(off, NULL);
#endif
                    if (    Addr > MaxPhysAddr
                        ||  !Addr
                        || (Addr & PAGE_OFFSET_MASK)
                        ||  (   fContiguous
                             && !off
                             && Addr == AddrPrev + PAGE_SIZE))
                    {
                        /* Buggy API, try allocate the memory another way. */
                        pMemDesc->complete();
                        pMemDesc->release();
                        if (PhysMask)
                        {
                            kprintf("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x - buggy API!\n",
                                    (size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions);
                            LogRel(("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x - buggy API!\n",
                                    (size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions));
                        }
                        return VERR_ADDRESS_TOO_BIG;
                    }
                    AddrPrev = Addr;
                }

                /*
                 * Check that it's aligned correctly.
                 */
                if ((uintptr_t)pv & (uAlignment - 1))
                {
                    pMemDesc->complete();
                    pMemDesc->release();
                    if (PhysMask)
                    {
                        kprintf("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x) - buggy API!!\n",
                                pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions);
                        LogRel(("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x) - buggy API!\n",
                                pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions));
                    }
                    return VERR_NOT_SUPPORTED;
                }

#ifdef RT_STRICT
                /* check that the memory is actually mapped. */
                //addr64_t Addr = pMemDesc->getPhysicalSegment64(0, NULL);
                //printf("rtR0MemObjNativeAllocWorker: pv=%p %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr);
                RTTHREADPREEMPTSTATE State = RTTHREADPREEMPTSTATE_INITIALIZER;
                RTThreadPreemptDisable(&State);
                rtR0MemObjDarwinTouchPages(pv, cb);
                RTThreadPreemptRestore(&State);
#endif

                /*
                 * Create the IPRT memory object.
                 */
                PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), enmType, pv, cb, pszTag);
                if (pMemDarwin)
                {
                    if (fOptions & kIOMemoryKernelUserShared)
                        pMemDarwin->Core.fFlags |= RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC;
                    else
                        pMemDarwin->Core.fFlags |= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC;
                    if (fContiguous)
                    {
#ifdef __LP64__
                        addr64_t PhysBase64 = pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone);
#else
                        addr64_t PhysBase64 = pMemDesc->getPhysicalSegment64(0, NULL);
#endif
                        RTHCPHYS PhysBase = PhysBase64; Assert(PhysBase == PhysBase64);
                        if (enmType == RTR0MEMOBJTYPE_CONT)
                            pMemDarwin->Core.u.Cont.Phys = PhysBase;
                        else if (enmType == RTR0MEMOBJTYPE_PHYS)
                            pMemDarwin->Core.u.Phys.PhysBase = PhysBase;
                        else
                            AssertMsgFailed(("enmType=%d\n", enmType));
                    }

                    if (fExecutable)
                    {
                        rc = rtR0MemObjNativeProtectWorker(&pMemDarwin->Core, 0, cb,
                                                           RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC);
#ifdef RT_STRICT
                        if (RT_SUCCESS(rc))
                        {
                            /* check that the memory is actually mapped. */
                            RTTHREADPREEMPTSTATE State2 = RTTHREADPREEMPTSTATE_INITIALIZER;
                            RTThreadPreemptDisable(&State2);
                            rtR0MemObjDarwinTouchPages(pv, cb);
                            RTThreadPreemptRestore(&State2);
                        }
#endif
                        /* Bug 6226: Ignore KERN_PROTECTION_FAILURE on Leopard and older. */
                        if (   rc == VERR_PERMISSION_DENIED
                            && version_major <= 10 /* 10 = 10.6.x = Snow Leopard. */)
                            rc = VINF_SUCCESS;
                    }
                    else
                        rc = VINF_SUCCESS;
                    if (RT_SUCCESS(rc))
                    {
                        pMemDarwin->pMemDesc = pMemDesc;
                        *ppMem = &pMemDarwin->Core;
                        return VINF_SUCCESS;
                    }

                    rtR0MemObjDelete(&pMemDarwin->Core);
                }

                if (enmType == RTR0MEMOBJTYPE_PHYS_NC)
                    rc = VERR_NO_PHYS_MEMORY;
                else if (enmType == RTR0MEMOBJTYPE_LOW)
                    rc = VERR_NO_LOW_MEMORY;
                else if (enmType == RTR0MEMOBJTYPE_CONT)
                    rc = VERR_NO_CONT_MEMORY;
                else
                    rc = VERR_NO_MEMORY;
            }
            else
                rc = VERR_MEMOBJ_INIT_FAILED;

            pMemDesc->complete();
        }
        else
            rc = RTErrConvertFromDarwinIO(IORet);
        pMemDesc->release();
    }
    else
        rc = VERR_MEMOBJ_INIT_FAILED;
    Assert(rc != VERR_ADDRESS_TOO_BIG);
    return rc;
}


/**
 * rtR0MemObjNativeAllocWorker kernel_task wrapper function.
 */
static void rtR0MemObjNativeAllockWorkerOnKernelThread(void *pvUser0, void *pvUser1)
{
    AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1);
    RTR0MEMOBJDARWINALLOCARGS volatile *pArgs = (RTR0MEMOBJDARWINALLOCARGS volatile *)pvUser0;
    int rc = rtR0MemObjNativeAllocWorker(pArgs->ppMem, pArgs->cb, pArgs->fExecutable, pArgs->fContiguous, pArgs->PhysMask,
                                         pArgs->MaxPhysAddr, pArgs->enmType, pArgs->uAlignment, pArgs->pszTag,
                                         true /*fOnKernelThread*/);
    rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc);
}


DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
{
    IPRT_DARWIN_SAVE_EFL_AC();

    int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, 0 /* PhysMask */, UINT64_MAX,
                                         RTR0MEMOBJTYPE_PAGE, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);

    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


DECLHIDDEN(int) rtR0MemObjNativeAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags,
                                           const char *pszTag)
{
    return rtR0MemObjFallbackAllocLarge(ppMem, cb, cbLargePage, fFlags, pszTag);
}


DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
{
    IPRT_DARWIN_SAVE_EFL_AC();

    /*
     * Try IOMallocPhysical/IOMallocAligned first.
     * Then try optimistically without a physical address mask, which will always
     * end up using IOMallocAligned.
     *
     * (See bug comment in the worker and IOBufferMemoryDescriptor::initWithPhysicalMask.)
     */
    int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, ~(uint32_t)PAGE_OFFSET_MASK,
                                         _4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);
    if (rc == VERR_ADDRESS_TOO_BIG)
        rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, 0 /* PhysMask */,
                                         _4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);

    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag)
{
    IPRT_DARWIN_SAVE_EFL_AC();

    int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, true /* fContiguous */,
                                         ~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE,
                                         RTR0MEMOBJTYPE_CONT, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);

    /*
     * Workaround for bogus IOKernelAllocateContiguous behavior, just in case.
     * cb <= PAGE_SIZE allocations take a different path, using a different allocator.
     */
    if (RT_FAILURE(rc) && cb <= PAGE_SIZE)
        rc = rtR0MemObjNativeAllocWorker(ppMem, cb + PAGE_SIZE, fExecutable, true /* fContiguous */,
                                         ~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE,
                                         RTR0MEMOBJTYPE_CONT, PAGE_SIZE, pszTag, false /*fOnKernelThread*/);
    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment,
                                          const char *pszTag)
{
    if (uAlignment != PAGE_SIZE)
    {
        /* See rtR0MemObjNativeAllocWorker: */
        if (version_major < 9 /* 9 = 10.5.x = Snow Leopard */)
            return VERR_NOT_SUPPORTED;
    }

    IPRT_DARWIN_SAVE_EFL_AC();

    /*
     * Translate the PhysHighest address into a mask.
     */
    int rc;
    if (PhysHighest == NIL_RTHCPHYS)
        rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable */, true /* fContiguous */,
                                         uAlignment <= PAGE_SIZE ? 0 : ~(mach_vm_address_t)(uAlignment - 1) /* PhysMask*/,
                                         UINT64_MAX, RTR0MEMOBJTYPE_PHYS, uAlignment, pszTag, false /*fOnKernelThread*/);
    else
    {
        mach_vm_address_t PhysMask = 0;
        PhysMask = ~(mach_vm_address_t)0;
        while (PhysMask > (PhysHighest | PAGE_OFFSET_MASK))
            PhysMask >>= 1;
        AssertReturn(PhysMask + 1 <= cb, VERR_INVALID_PARAMETER);
        PhysMask &= ~(mach_vm_address_t)(uAlignment - 1);

        rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable */, true /* fContiguous */,
                                         PhysMask, PhysHighest,
                                         RTR0MEMOBJTYPE_PHYS, uAlignment, pszTag, false /*fOnKernelThread*/);
    }

    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, const char *pszTag)
{
    /** @todo rtR0MemObjNativeAllocPhys / darwin.
     * This might be a bit problematic and may very well require having to create our own
     * object which we populate with pages but without mapping it into any address space.
     * Estimate is 2-3 days.
     */
    RT_NOREF(ppMem, cb, PhysHighest, pszTag);
    return VERR_NOT_SUPPORTED;
}


DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy,
                                          const char *pszTag)
{
    AssertReturn(uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE, VERR_NOT_SUPPORTED);
    IPRT_DARWIN_SAVE_EFL_AC();

    /*
     * Create a descriptor for it (the validation is always true on intel macs, but
     * as it doesn't harm us keep it in).
     */
    int rc = VERR_ADDRESS_TOO_BIG;
    IOAddressRange aRanges[1] = { { Phys, cb } };
    if (    aRanges[0].address == Phys
        &&  aRanges[0].length == cb)
    {
        IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges),
                                                                             kIODirectionInOut, NULL /*task*/);
        if (pMemDesc)
        {
#ifdef __LP64__
            Assert(Phys == pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone));
#else
            Assert(Phys == pMemDesc->getPhysicalSegment64(0, NULL));
#endif

            /*
             * Create the IPRT memory object.
             */
            PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_PHYS,
                                                                            NULL, cb, pszTag);
            if (pMemDarwin)
            {
                pMemDarwin->Core.u.Phys.PhysBase = Phys;
                pMemDarwin->Core.u.Phys.fAllocated = false;
                pMemDarwin->Core.u.Phys.uCachePolicy = uCachePolicy;
                pMemDarwin->pMemDesc = pMemDesc;
                *ppMem = &pMemDarwin->Core;
                IPRT_DARWIN_RESTORE_EFL_AC();
                return VINF_SUCCESS;
            }

            rc = VERR_NO_MEMORY;
            pMemDesc->release();
        }
        else
            rc = VERR_MEMOBJ_INIT_FAILED;
    }
    else
        AssertMsgFailed(("%#llx %llx\n", (unsigned long long)Phys, (unsigned long long)cb));
    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


/**
 * Internal worker for locking down pages.
 *
 * @return IPRT status code.
 *
 * @param   ppMem           Where to store the memory object pointer.
 * @param   pv              First page.
 * @param   cb              Number of bytes.
 * @param   fAccess         The desired access, a combination of RTMEM_PROT_READ
 *                          and RTMEM_PROT_WRITE.
 * @param   Task            The task \a pv and \a cb refers to.
 * @param   pszTag          Allocation tag used for statistics and such.
 */
static int rtR0MemObjNativeLock(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, task_t Task,
                                const char *pszTag)
{
    IPRT_DARWIN_SAVE_EFL_AC();
    NOREF(fAccess);
#ifdef USE_VM_MAP_WIRE
    vm_map_t Map = get_task_map(Task);
    Assert(Map);

    /*
     * First try lock the memory.
     */
    int rc = VERR_LOCK_FAILED;
    kern_return_t kr = vm_map_wire(get_task_map(Task),
                                   (vm_map_offset_t)pv,
                                   (vm_map_offset_t)pv + cb,
                                   VM_PROT_DEFAULT,
                                   0 /* not user */);
    if (kr == KERN_SUCCESS)
    {
        /*
         * Create the IPRT memory object.
         */
        PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK, pv, cb, pszTag);
        if (pMemDarwin)
        {
            pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task;
            *ppMem = &pMemDarwin->Core;

            IPRT_DARWIN_RESTORE_EFL_AC();
            return VINF_SUCCESS;
        }

        kr = vm_map_unwire(get_task_map(Task), (vm_map_offset_t)pv, (vm_map_offset_t)pv + cb, 0 /* not user */);
        Assert(kr == KERN_SUCCESS);
        rc = VERR_NO_MEMORY;
    }

#else

    /*
     * Create a descriptor and try lock it (prepare).
     */
    int rc = VERR_MEMOBJ_INIT_FAILED;
    IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRange((vm_address_t)pv, cb, kIODirectionInOut, Task);
    if (pMemDesc)
    {
        IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
        if (IORet == kIOReturnSuccess)
        {
            /*
             * Create the IPRT memory object.
             */
            PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK,
                                                                            pv, cb, pszTag);
            if (pMemDarwin)
            {
                pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task;
                pMemDarwin->pMemDesc = pMemDesc;
                *ppMem = &pMemDarwin->Core;

                IPRT_DARWIN_RESTORE_EFL_AC();
                return VINF_SUCCESS;
            }

            pMemDesc->complete();
            rc = VERR_NO_MEMORY;
        }
        else
            rc = VERR_LOCK_FAILED;
        pMemDesc->release();
    }
#endif
    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess,
                                         RTR0PROCESS R0Process, const char *pszTag)
{
    return rtR0MemObjNativeLock(ppMem, (void *)R3Ptr, cb, fAccess, (task_t)R0Process, pszTag);
}


DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, const char *pszTag)
{
    return rtR0MemObjNativeLock(ppMem, pv, cb, fAccess, kernel_task, pszTag);
}


DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment,
                                              const char *pszTag)
{
    RT_NOREF(ppMem, pvFixed, cb, uAlignment, pszTag);
    return VERR_NOT_SUPPORTED;
}


DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment,
                                            RTR0PROCESS R0Process, const char *pszTag)
{
    RT_NOREF(ppMem, R3PtrFixed, cb, uAlignment, R0Process, pszTag);
    return VERR_NOT_SUPPORTED;
}


DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment,
                                          unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag)
{
    RT_NOREF(fProt);
    AssertReturn(pvFixed == (void *)-1, VERR_NOT_SUPPORTED);

    /*
     * Check that the specified alignment is supported.
     */
    if (uAlignment > PAGE_SIZE)
        return VERR_NOT_SUPPORTED;
    Assert(!offSub || cbSub);

    IPRT_DARWIN_SAVE_EFL_AC();

    /*
     * Must have a memory descriptor that we can map.
     */
    int rc = VERR_INVALID_PARAMETER;
    PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap;
    if (pMemToMapDarwin->pMemDesc)
    {
        /* The kIOMapPrefault option was added in 10.10.0; causes PTEs to be populated with
           INTEL_PTE_WIRED to be set, just like we desire (see further down).  However, till
           10.13.0 it was not available for use on kernel mappings. Oh, fudge. */
#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
        static uint32_t volatile s_fOptions = UINT32_MAX;
        uint32_t fOptions = s_fOptions;
        if (RT_UNLIKELY(fOptions == UINT32_MAX))
            s_fOptions = fOptions = version_major >= 17 ? 0x10000000 /*kIOMapPrefault*/ : 0; /* Since 10.13.0 (High Sierra). */

        IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask(kernel_task,
                                                                              0,
                                                                              kIOMapAnywhere | kIOMapDefaultCache | fOptions,
                                                                              offSub,
                                                                              cbSub);
#else
        IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map(kernel_task,
                                                              0,
                                                              kIOMapAnywhere | kIOMapDefaultCache,
                                                              offSub,
                                                              cbSub);
#endif
        if (pMemMap)
        {
            IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress();
            void *pv = (void *)(uintptr_t)VirtAddr;
            if ((uintptr_t)pv == VirtAddr && pv != NULL)
            {
//#ifdef __LP64__
//                addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone);
//#else
//                addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL);
//#endif
//                MY_PRINTF("pv=%p: %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr);

//                /*
//                 * Explicitly lock it so that we're sure it is present and that
//                 * its PTEs cannot be recycled.
//                 * Note! withAddressRange() doesn't work as it adds kIOMemoryTypeVirtual64
//                 *       to the options which causes prepare() to not wire the pages.
//                 *       This is probably a bug.
//                 */
//                IOAddressRange Range = { (mach_vm_address_t)pv, cbSub };
//                IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withOptions(&Range,
//                                                                               1 /* count */,
//                                                                               0 /* offset */,
//                                                                               kernel_task,
//                                                                               kIODirectionInOut | kIOMemoryTypeVirtual,
//                                                                               kIOMapperSystem);
//                if (pMemDesc)
//                {
//                    IOReturn IORet = pMemDesc->prepare(kIODirectionInOut);
//                    if (IORet == kIOReturnSuccess)
//                    {
                        /* HACK ALERT! On kernels older than 10.10 (xnu version 14), we need to fault in
                                       the pages here so they can safely be accessed from inside simple
                                       locks and when preemption is disabled (no page-ins allowed).
                           Note! This touching does not cause INTEL_PTE_WIRED (bit 10) to be set as we go
                                 thru general #PF and vm_fault doesn't figure it should be wired or something.  */
                        rtR0MemObjDarwinTouchPages(pv, cbSub ? cbSub : pMemToMap->cb);
                        /** @todo First, the memory should've been mapped by now, and second, it
                         *        should have the wired attribute in the PTE (bit 10). Neither seems to
                         *        be the case. The disabled locking code doesn't make any difference,
                         *        which is extremely odd, and breaks rtR0MemObjNativeGetPagePhysAddr
                         *        (getPhysicalSegment64 -> 64 for the lock descriptor. */
//#ifdef __LP64__
//                        addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone);
//#else
//                        addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL);
//#endif
//                        MY_PRINTF("pv=%p: %8llx %8llx (%d)\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr2, 2);

                        /*
                         * Create the IPRT memory object.
                         */
                        PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING,
                                                                                        pv, cbSub ? cbSub : pMemToMap->cb, pszTag);
                        if (pMemDarwin)
                        {
                            pMemDarwin->Core.u.Mapping.R0Process = NIL_RTR0PROCESS;
                            pMemDarwin->pMemMap = pMemMap;
//                            pMemDarwin->pMemDesc = pMemDesc;
                            *ppMem = &pMemDarwin->Core;

                            IPRT_DARWIN_RESTORE_EFL_AC();
                            return VINF_SUCCESS;
                        }

//                        pMemDesc->complete();
//                        rc = VERR_NO_MEMORY;
//                    }
//                    else
//                        rc = RTErrConvertFromDarwinIO(IORet);
//                    pMemDesc->release();
//                }
//                else
//                    rc = VERR_MEMOBJ_INIT_FAILED;
            }
            else if (pv)
                rc = VERR_ADDRESS_TOO_BIG;
            else
                rc = VERR_MAP_FAILED;
            pMemMap->release();
        }
        else
            rc = VERR_MAP_FAILED;
    }

    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment,
                                        unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag)
{
    RT_NOREF(fProt);

    /*
     * Check for unsupported things.
     */
    AssertReturn(R3PtrFixed == (RTR3PTR)-1, VERR_NOT_SUPPORTED);
    if (uAlignment > PAGE_SIZE)
        return VERR_NOT_SUPPORTED;
    Assert(!offSub || cbSub);

    IPRT_DARWIN_SAVE_EFL_AC();

    /*
     * Must have a memory descriptor.
     */
    int rc = VERR_INVALID_PARAMETER;
    PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap;
    if (pMemToMapDarwin->pMemDesc)
    {
#if MAC_OS_X_VERSION_MIN_REQUIRED >= 101000 /* The kIOMapPrefault option was added in 10.10.0. */
         IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process,
                                                                              0,
                                                                              kIOMapAnywhere | kIOMapDefaultCache | kIOMapPrefault,
                                                                              offSub,
                                                                              cbSub);
#elif MAC_OS_X_VERSION_MIN_REQUIRED >= 1050
        static uint32_t volatile s_fOptions = UINT32_MAX;
        uint32_t fOptions = s_fOptions;
        if (RT_UNLIKELY(fOptions == UINT32_MAX))
            s_fOptions = fOptions = version_major >= 14 ? 0x10000000 /*kIOMapPrefault*/ : 0; /* Since 10.10.0. */
        IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process,
                                                                              0,
                                                                              kIOMapAnywhere | kIOMapDefaultCache | fOptions,
                                                                              offSub,
                                                                              cbSub);
#else
        IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map((task_t)R0Process,
                                                              0,
                                                              kIOMapAnywhere | kIOMapDefaultCache,
                                                              offSub,
                                                              cbSub);
#endif
        if (pMemMap)
        {
            IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress();
            void *pv = (void *)(uintptr_t)VirtAddr;
            if ((uintptr_t)pv == VirtAddr && pv != NULL)
            {
                /*
                 * Create the IPRT memory object.
                 */
                PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING,
                                                                                pv, cbSub ? cbSub : pMemToMap->cb, pszTag);
                if (pMemDarwin)
                {
                    pMemDarwin->Core.u.Mapping.R0Process = R0Process;
                    pMemDarwin->pMemMap = pMemMap;
                    *ppMem = &pMemDarwin->Core;

                    IPRT_DARWIN_RESTORE_EFL_AC();
                    return VINF_SUCCESS;
                }

                rc = VERR_NO_MEMORY;
            }
            else if (pv)
                rc = VERR_ADDRESS_TOO_BIG;
            else
                rc = VERR_MAP_FAILED;
            pMemMap->release();
        }
        else
            rc = VERR_MAP_FAILED;
    }

    IPRT_DARWIN_RESTORE_EFL_AC();
    return rc;
}


/**
 * Worker for rtR0MemObjNativeProtect that's typically called in a different
 * context.
 */
static int rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
{
    IPRT_DARWIN_SAVE_EFL_AC();

    /* Get the map for the object. */
    vm_map_t pVmMap = rtR0MemObjDarwinGetMap(pMem);
    if (!pVmMap)
    {
        IPRT_DARWIN_RESTORE_EFL_AC();
        return VERR_NOT_SUPPORTED;
    }

    /*
     * Convert the protection.
     */
    vm_prot_t fMachProt;
    switch (fProt)
    {
        case RTMEM_PROT_NONE:
            fMachProt = VM_PROT_NONE;
            break;
        case RTMEM_PROT_READ:
            fMachProt = VM_PROT_READ;
            break;
        case RTMEM_PROT_READ | RTMEM_PROT_WRITE:
            fMachProt = VM_PROT_READ | VM_PROT_WRITE;
            break;
        case RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC:
            fMachProt = VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
            break;
        case RTMEM_PROT_WRITE:
            fMachProt = VM_PROT_WRITE | VM_PROT_READ;                   /* never write-only */
            break;
        case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC:
            fMachProt = VM_PROT_WRITE | VM_PROT_EXECUTE | VM_PROT_READ; /* never write-only or execute-only */
            break;
        case RTMEM_PROT_EXEC:
            fMachProt = VM_PROT_EXECUTE | VM_PROT_READ;                 /* never execute-only */
            break;
        default:
            AssertFailedReturn(VERR_INVALID_PARAMETER);
    }

    /*
     * Do the job.
     */
    vm_offset_t Start = (uintptr_t)pMem->pv + offSub;
    kern_return_t krc = vm_protect(pVmMap,
                                   Start,
                                   cbSub,
                                   false,
                                   fMachProt);
    if (krc != KERN_SUCCESS)
    {
        static int s_cComplaints = 0;
        if (s_cComplaints < 10)
        {
            s_cComplaints++;
            printf("rtR0MemObjNativeProtect: vm_protect(%p,%p,%p,false,%#x) -> %d\n",
                   (void *)pVmMap, (void *)Start, (void *)cbSub, fMachProt, krc);

            kern_return_t               krc2;
            vm_offset_t                 pvReal = Start;
            vm_size_t                   cbReal = 0;
            mach_msg_type_number_t      cInfo  = VM_REGION_BASIC_INFO_COUNT;
            struct vm_region_basic_info Info;
            RT_ZERO(Info);
            krc2 = vm_region(pVmMap, &pvReal, &cbReal, VM_REGION_BASIC_INFO, (vm_region_info_t)&Info, &cInfo, NULL);
            printf("rtR0MemObjNativeProtect: basic info - krc2=%d pv=%p cb=%p prot=%#x max=%#x inh=%#x shr=%d rvd=%d off=%#x behavior=%#x wired=%#x\n",
                   krc2, (void *)pvReal, (void *)cbReal, Info.protection, Info.max_protection,  Info.inheritance,
                   Info.shared, Info.reserved, Info.offset, Info.behavior, Info.user_wired_count);
        }
        IPRT_DARWIN_RESTORE_EFL_AC();
        return RTErrConvertFromDarwinKern(krc);
    }

    /*
     * Touch the pages if they should be writable afterwards and accessible
     * from code which should never fault. vm_protect() may leave pages
     * temporarily write protected, possibly due to pmap no-upgrade rules?
     *
     * This is the same trick (or HACK ALERT if you like) as applied in
     * rtR0MemObjNativeMapKernel.
     */
    if (   pMem->enmType != RTR0MEMOBJTYPE_MAPPING
        || pMem->u.Mapping.R0Process == NIL_RTR0PROCESS)
    {
        if (fProt & RTMEM_PROT_WRITE)
            rtR0MemObjDarwinTouchPages((void *)Start, cbSub);
        /*
         * Sniff (read) read-only pages too, just to be sure.
         */
        else if (fProt & (RTMEM_PROT_READ | RTMEM_PROT_EXEC))
            rtR0MemObjDarwinSniffPages((void const *)Start, cbSub);
    }

    IPRT_DARWIN_RESTORE_EFL_AC();
    return VINF_SUCCESS;
}


/**
 * rtR0MemObjNativeProtect kernel_task wrapper function.
 */
static void rtR0MemObjNativeProtectWorkerOnKernelThread(void *pvUser0, void *pvUser1)
{
    AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1);
    RTR0MEMOBJDARWINPROTECTARGS *pArgs = (RTR0MEMOBJDARWINPROTECTARGS *)pvUser0;
    int rc = rtR0MemObjNativeProtectWorker(pArgs->pMem, pArgs->offSub, pArgs->cbSub, pArgs->fProt);
    rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc);
}


DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt)
{
    /*
     * The code won't work right because process codesigning properties leaks
     * into kernel_map memory management.  So, if the user process we're running
     * in has CS restrictions active, we cannot play around with the EXEC
     * protection because some vm_fault.c think we're modifying the process map
     * or something.
     */
    int rc;
    if (rtR0MemObjDarwinGetMap(pMem) == kernel_map)
    {
        RTR0MEMOBJDARWINPROTECTARGS Args;
        Args.pMem       = pMem;
        Args.offSub     = offSub;
        Args.cbSub      = cbSub;
        Args.fProt      = fProt;
        rc = rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeProtectWorkerOnKernelThread, &Args.Core);
    }
    else
        rc = rtR0MemObjNativeProtectWorker(pMem, offSub, cbSub, fProt);
    return rc;
}


DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
{
    RTHCPHYS            PhysAddr;
    PRTR0MEMOBJDARWIN   pMemDarwin = (PRTR0MEMOBJDARWIN)pMem;
    IPRT_DARWIN_SAVE_EFL_AC();

#ifdef USE_VM_MAP_WIRE
    /*
     * Locked memory doesn't have a memory descriptor and
     * needs to be handled differently.
     */
    if (pMemDarwin->Core.enmType == RTR0MEMOBJTYPE_LOCK)
    {
        ppnum_t PgNo;
        if (pMemDarwin->Core.u.Lock.R0Process == NIL_RTR0PROCESS)
            PgNo = pmap_find_phys(kernel_pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE);
        else
        {
            /*
             * From what I can tell, Apple seems to have locked up the all the
             * available interfaces that could help us obtain the pmap_t of a task
             * or vm_map_t.

             * So, we'll have to figure out where in the vm_map_t  structure it is
             * and read it our selves. ASSUMING that kernel_pmap is pointed to by
             * kernel_map->pmap, we scan kernel_map to locate the structure offset.
             * Not nice, but it will hopefully do the job in a reliable manner...
             *
             * (get_task_pmap, get_map_pmap or vm_map_pmap is what we really need btw.)
             */
            static int s_offPmap = -1;
            if (RT_UNLIKELY(s_offPmap == -1))
            {
                pmap_t const *p = (pmap_t *)kernel_map;
                pmap_t const * const pEnd = p + 64;
                for (; p < pEnd; p++)
                    if (*p == kernel_pmap)
                    {
                        s_offPmap = (uintptr_t)p - (uintptr_t)kernel_map;
                        break;
                    }
                AssertReturn(s_offPmap >= 0, NIL_RTHCPHYS);
            }
            pmap_t Pmap = *(pmap_t *)((uintptr_t)get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process) + s_offPmap);
            PgNo = pmap_find_phys(Pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE);
        }

        IPRT_DARWIN_RESTORE_EFL_AC();
        AssertReturn(PgNo, NIL_RTHCPHYS);
        PhysAddr = (RTHCPHYS)PgNo << PAGE_SHIFT;
        Assert((PhysAddr >> PAGE_SHIFT) == PgNo);
    }
    else
#endif /* USE_VM_MAP_WIRE */
    {
        /*
         * Get the memory descriptor.
         */
        IOMemoryDescriptor *pMemDesc = pMemDarwin->pMemDesc;
        if (!pMemDesc)
            pMemDesc = pMemDarwin->pMemMap->getMemoryDescriptor();
        AssertReturn(pMemDesc, NIL_RTHCPHYS);

        /*
         * If we've got a memory descriptor, use getPhysicalSegment64().
         */
#ifdef __LP64__
        addr64_t Addr = pMemDesc->getPhysicalSegment(iPage * PAGE_SIZE, NULL, kIOMemoryMapperNone);
#else
        addr64_t Addr = pMemDesc->getPhysicalSegment64(iPage * PAGE_SIZE, NULL);
#endif
        IPRT_DARWIN_RESTORE_EFL_AC();
        AssertMsgReturn(Addr, ("iPage=%u\n", iPage), NIL_RTHCPHYS);
        PhysAddr = Addr;
        AssertMsgReturn(PhysAddr == Addr, ("PhysAddr=%RHp Addr=%RX64\n", PhysAddr, (uint64_t)Addr), NIL_RTHCPHYS);
    }

    return PhysAddr;
}