diff options
Diffstat (limited to 'src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp')
| -rw-r--r-- | src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp | 1571 |
1 files changed, 1571 insertions, 0 deletions
diff --git a/src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp b/src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp new file mode 100644 index 00000000..57fb4bee --- /dev/null +++ b/src/VBox/Runtime/r0drv/darwin/memobj-r0drv-darwin.cpp @@ -0,0 +1,1571 @@ +/* $Id: memobj-r0drv-darwin.cpp $ */ +/** @file + * IPRT - Ring-0 Memory Objects, Darwin. + */ + +/* + * Copyright (C) 2006-2022 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <https://www.gnu.org/licenses>. + * + * The contents of this file may alternatively be used under the terms + * of the Common Development and Distribution License Version 1.0 + * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included + * in the VirtualBox distribution, in which case the provisions of the + * CDDL are applicable instead of those of the GPL. + * + * You may elect to license modified versions of this file under the + * terms and conditions of either the GPL or the CDDL or both. + * + * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define RTMEM_NO_WRAP_TO_EF_APIS /* circular dependency otherwise. */ +#include "the-darwin-kernel.h" +#include "internal/iprt.h" +#include <iprt/memobj.h> + +#include <iprt/asm.h> +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include <iprt/x86.h> +# include <iprt/asm-amd64-x86.h> +#endif +#include <iprt/assert.h> +#include <iprt/log.h> +#include <iprt/mem.h> +#include <iprt/param.h> +#include <iprt/process.h> +#include <iprt/semaphore.h> +#include <iprt/string.h> +#include <iprt/thread.h> +#include "internal/memobj.h" + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#define MY_PRINTF(...) do { printf(__VA_ARGS__); kprintf(__VA_ARGS__); } while (0) + +/*#define USE_VM_MAP_WIRE - may re-enable later when non-mapped allocations are added. */ + + +/********************************************************************************************************************************* +* Structures and Typedefs * +*********************************************************************************************************************************/ +/** + * The Darwin version of the memory object structure. + */ +typedef struct RTR0MEMOBJDARWIN +{ + /** The core structure. */ + RTR0MEMOBJINTERNAL Core; + /** Pointer to the memory descriptor created for allocated and locked memory. */ + IOMemoryDescriptor *pMemDesc; + /** Pointer to the memory mapping object for mapped memory. */ + IOMemoryMap *pMemMap; +} RTR0MEMOBJDARWIN, *PRTR0MEMOBJDARWIN; + +/** + * Common thread_call_allocate/thread_call_enter argument package. + */ +typedef struct RTR0MEMOBJDARWINTHREADARGS +{ + int32_t volatile rc; + RTSEMEVENTMULTI hEvent; +} RTR0MEMOBJDARWINTHREADARGS; + + +/** + * Arguments for rtR0MemObjNativeAllockWorkOnKernelThread. + */ +typedef struct RTR0MEMOBJDARWINALLOCARGS +{ + RTR0MEMOBJDARWINTHREADARGS Core; + PPRTR0MEMOBJINTERNAL ppMem; + size_t cb; + bool fExecutable; + bool fContiguous; + mach_vm_address_t PhysMask; + uint64_t MaxPhysAddr; + RTR0MEMOBJTYPE enmType; + size_t uAlignment; + const char *pszTag; +} RTR0MEMOBJDARWINALLOCARGS; + +/** + * Arguments for rtR0MemObjNativeProtectWorkOnKernelThread. + */ +typedef struct RTR0MEMOBJDARWINPROTECTARGS +{ + RTR0MEMOBJDARWINTHREADARGS Core; + PRTR0MEMOBJINTERNAL pMem; + size_t offSub; + size_t cbSub; + uint32_t fProt; +} RTR0MEMOBJDARWINPROTECTARGS; + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +static void rtR0MemObjNativeAllockWorkerOnKernelThread(void *pvUser0, void *pvUser1); +static int rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt); +static void rtR0MemObjNativeProtectWorkerOnKernelThread(void *pvUser0, void *pvUser1); + + +/** + * Touch the pages to force the kernel to create or write-enable the page table + * entries. + * + * This is necessary since the kernel gets upset if we take a page fault when + * preemption is disabled and/or we own a simple lock (same thing). It has no + * problems with us disabling interrupts when taking the traps, weird stuff. + * + * (This is basically a way of invoking vm_fault on a range of pages.) + * + * @param pv Pointer to the first page. + * @param cb The number of bytes. + */ +static void rtR0MemObjDarwinTouchPages(void *pv, size_t cb) +{ + uint32_t volatile *pu32 = (uint32_t volatile *)pv; + for (;;) + { + ASMAtomicCmpXchgU32(pu32, 0xdeadbeef, 0xdeadbeef); + if (cb <= PAGE_SIZE) + break; + cb -= PAGE_SIZE; + pu32 += PAGE_SIZE / sizeof(uint32_t); + } +} + + +/** + * Read (sniff) every page in the range to make sure there are some page tables + * entries backing it. + * + * This is just to be sure vm_protect didn't remove stuff without re-adding it + * if someone should try write-protect something. + * + * @param pv Pointer to the first page. + * @param cb The number of bytes. + */ +static void rtR0MemObjDarwinSniffPages(void const *pv, size_t cb) +{ + uint32_t volatile *pu32 = (uint32_t volatile *)pv; + uint32_t volatile u32Counter = 0; + for (;;) + { + u32Counter += *pu32; + + if (cb <= PAGE_SIZE) + break; + cb -= PAGE_SIZE; + pu32 += PAGE_SIZE / sizeof(uint32_t); + } +} + + +/** + * Gets the virtual memory map the specified object is mapped into. + * + * @returns VM map handle on success, NULL if no map. + * @param pMem The memory object. + */ +DECLINLINE(vm_map_t) rtR0MemObjDarwinGetMap(PRTR0MEMOBJINTERNAL pMem) +{ + switch (pMem->enmType) + { + case RTR0MEMOBJTYPE_PAGE: + case RTR0MEMOBJTYPE_LOW: + case RTR0MEMOBJTYPE_CONT: + return kernel_map; + + case RTR0MEMOBJTYPE_PHYS: + case RTR0MEMOBJTYPE_PHYS_NC: + if (pMem->pv) + return kernel_map; + return NULL; + + case RTR0MEMOBJTYPE_LOCK: + return pMem->u.Lock.R0Process == NIL_RTR0PROCESS + ? kernel_map + : get_task_map((task_t)pMem->u.Lock.R0Process); + + case RTR0MEMOBJTYPE_RES_VIRT: + return pMem->u.ResVirt.R0Process == NIL_RTR0PROCESS + ? kernel_map + : get_task_map((task_t)pMem->u.ResVirt.R0Process); + + case RTR0MEMOBJTYPE_MAPPING: + return pMem->u.Mapping.R0Process == NIL_RTR0PROCESS + ? kernel_map + : get_task_map((task_t)pMem->u.Mapping.R0Process); + + default: + return NULL; + } +} + +#if 0 /* not necessary after all*/ +/* My vm_map mockup. */ +struct my_vm_map +{ + struct { char pad[8]; } lock; + struct my_vm_map_header + { + struct vm_map_links + { + void *prev; + void *next; + vm_map_offset_t start; + vm_map_offset_t end; + } links; + int nentries; + boolean_t entries_pageable; + } hdr; + pmap_t pmap; + vm_map_size_t size; +}; + + +/** + * Gets the minimum map address, this is similar to get_map_min. + * + * @returns The start address of the map. + * @param pMap The map. + */ +static vm_map_offset_t rtR0MemObjDarwinGetMapMin(vm_map_t pMap) +{ + /* lazy discovery of the correct offset. The apple guys is a wonderfully secretive bunch. */ + static int32_t volatile s_offAdjust = INT32_MAX; + int32_t off = s_offAdjust; + if (off == INT32_MAX) + { + for (off = 0; ; off += sizeof(pmap_t)) + { + if (*(pmap_t *)((uint8_t *)kernel_map + off) == kernel_pmap) + break; + AssertReturn(off <= RT_MAX(RT_OFFSETOF(struct my_vm_map, pmap) * 4, 1024), 0x1000); + } + ASMAtomicWriteS32(&s_offAdjust, off - RT_OFFSETOF(struct my_vm_map, pmap)); + } + + /* calculate it. */ + struct my_vm_map *pMyMap = (struct my_vm_map *)((uint8_t *)pMap + off); + return pMyMap->hdr.links.start; +} +#endif /* unused */ + +#ifdef RT_STRICT +# if 0 /* unused */ + +/** + * Read from a physical page. + * + * @param HCPhys The address to start reading at. + * @param cb How many bytes to read. + * @param pvDst Where to put the bytes. This is zero'd on failure. + */ +static void rtR0MemObjDarwinReadPhys(RTHCPHYS HCPhys, size_t cb, void *pvDst) +{ + memset(pvDst, '\0', cb); + + IOAddressRange aRanges[1] = { { (mach_vm_address_t)HCPhys, RT_ALIGN_Z(cb, PAGE_SIZE) } }; + IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges), + kIODirectionIn, NULL /*task*/); + if (pMemDesc) + { +#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050 + IOMemoryMap *pMemMap = pMemDesc->createMappingInTask(kernel_task, 0, kIOMapAnywhere | kIOMapDefaultCache); +#else + IOMemoryMap *pMemMap = pMemDesc->map(kernel_task, 0, kIOMapAnywhere | kIOMapDefaultCache); +#endif + if (pMemMap) + { + void const *pvSrc = (void const *)(uintptr_t)pMemMap->getVirtualAddress(); + memcpy(pvDst, pvSrc, cb); + pMemMap->release(); + } + else + MY_PRINTF("rtR0MemObjDarwinReadPhys: createMappingInTask failed; HCPhys=%llx\n", HCPhys); + + pMemDesc->release(); + } + else + MY_PRINTF("rtR0MemObjDarwinReadPhys: withAddressRanges failed; HCPhys=%llx\n", HCPhys); +} + + +/** + * Gets the PTE for a page. + * + * @returns the PTE. + * @param pvPage The virtual address to get the PTE for. + */ +static uint64_t rtR0MemObjDarwinGetPTE(void *pvPage) +{ + RTUINT64U u64; + RTCCUINTREG cr3 = ASMGetCR3(); + RTCCUINTREG cr4 = ASMGetCR4(); + bool fPAE = false; + bool fLMA = false; + if (cr4 & X86_CR4_PAE) + { + fPAE = true; + uint32_t fExtFeatures = ASMCpuId_EDX(0x80000001); + if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE) + { + uint64_t efer = ASMRdMsr(MSR_K6_EFER); + if (efer & MSR_K6_EFER_LMA) + fLMA = true; + } + } + + if (fLMA) + { + /* PML4 */ + rtR0MemObjDarwinReadPhys((cr3 & ~(RTCCUINTREG)PAGE_OFFSET_MASK) | (((uint64_t)(uintptr_t)pvPage >> X86_PML4_SHIFT) & X86_PML4_MASK) * 8, 8, &u64); + if (!(u64.u & X86_PML4E_P)) + { + MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PML4E !p\n", pvPage); + return 0; + } + + /* PDPTR */ + rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64) * 8, 8, &u64); + if (!(u64.u & X86_PDPE_P)) + { + MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDPTE !p\n", pvPage); + return 0; + } + if (u64.u & X86_PDPE_LM_PS) + return (u64.u & ~(uint64_t)(_1G -1)) | ((uintptr_t)pvPage & (_1G -1)); + + /* PD */ + rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64); + if (!(u64.u & X86_PDE_P)) + { + MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PDE !p\n", pvPage); + return 0; + } + if (u64.u & X86_PDE_PS) + return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1)); + + /* PT */ + rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64); + if (!(u64.u & X86_PTE_P)) + { + MY_PRINTF("rtR0MemObjDarwinGetPTE: %p -> PTE !p\n", pvPage); + return 0; + } + return u64.u; + } + + if (fPAE) + { + /* PDPTR */ + rtR0MemObjDarwinReadPhys((u64.u & X86_CR3_PAE_PAGE_MASK) | (((uintptr_t)pvPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE) * 8, 8, &u64); + if (!(u64.u & X86_PDE_P)) + return 0; + + /* PD */ + rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK) * 8, 8, &u64); + if (!(u64.u & X86_PDE_P)) + return 0; + if (u64.u & X86_PDE_PS) + return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1)); + + /* PT */ + rtR0MemObjDarwinReadPhys((u64.u & ~(uint64_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK) * 8, 8, &u64); + if (!(u64.u & X86_PTE_P)) + return 0; + return u64.u; + } + + /* PD */ + rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PD_SHIFT) & X86_PD_MASK) * 4, 4, &u64); + if (!(u64.au32[0] & X86_PDE_P)) + return 0; + if (u64.au32[0] & X86_PDE_PS) + return (u64.u & ~(uint64_t)(_2M -1)) | ((uintptr_t)pvPage & (_2M -1)); + + /* PT */ + rtR0MemObjDarwinReadPhys((u64.au32[0] & ~(uint32_t)PAGE_OFFSET_MASK) | (((uintptr_t)pvPage >> X86_PT_SHIFT) & X86_PT_MASK) * 4, 4, &u64); + if (!(u64.au32[0] & X86_PTE_P)) + return 0; + return u64.au32[0]; + + return 0; +} + +# endif /* unused */ +#endif /* RT_STRICT */ + +DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem) +{ + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)pMem; + IPRT_DARWIN_SAVE_EFL_AC(); + + /* + * Release the IOMemoryDescriptor or/and IOMemoryMap associated with the object. + */ + if (pMemDarwin->pMemDesc) + { + pMemDarwin->pMemDesc->complete(); + pMemDarwin->pMemDesc->release(); + pMemDarwin->pMemDesc = NULL; + } + + if (pMemDarwin->pMemMap) + { + pMemDarwin->pMemMap->release(); + pMemDarwin->pMemMap = NULL; + } + + /* + * Release any memory that we've allocated or locked. + */ + switch (pMemDarwin->Core.enmType) + { + case RTR0MEMOBJTYPE_LOW: + case RTR0MEMOBJTYPE_PAGE: + case RTR0MEMOBJTYPE_CONT: + break; + + case RTR0MEMOBJTYPE_LOCK: + { +#ifdef USE_VM_MAP_WIRE + vm_map_t Map = pMemDarwin->Core.u.Lock.R0Process != NIL_RTR0PROCESS + ? get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process) + : kernel_map; + kern_return_t kr = vm_map_unwire(Map, + (vm_map_offset_t)pMemDarwin->Core.pv, + (vm_map_offset_t)pMemDarwin->Core.pv + pMemDarwin->Core.cb, + 0 /* not user */); + AssertRC(kr == KERN_SUCCESS); /** @todo don't ignore... */ +#endif + break; + } + + case RTR0MEMOBJTYPE_PHYS: + /*if (pMemDarwin->Core.u.Phys.fAllocated) + IOFreePhysical(pMemDarwin->Core.u.Phys.PhysBase, pMemDarwin->Core.cb);*/ + Assert(!pMemDarwin->Core.u.Phys.fAllocated); + break; + + case RTR0MEMOBJTYPE_PHYS_NC: + AssertMsgFailed(("RTR0MEMOBJTYPE_PHYS_NC\n")); + IPRT_DARWIN_RESTORE_EFL_AC(); + return VERR_INTERNAL_ERROR; + + case RTR0MEMOBJTYPE_RES_VIRT: + AssertMsgFailed(("RTR0MEMOBJTYPE_RES_VIRT\n")); + IPRT_DARWIN_RESTORE_EFL_AC(); + return VERR_INTERNAL_ERROR; + + case RTR0MEMOBJTYPE_MAPPING: + /* nothing to do here. */ + break; + + default: + AssertMsgFailed(("enmType=%d\n", pMemDarwin->Core.enmType)); + IPRT_DARWIN_RESTORE_EFL_AC(); + return VERR_INTERNAL_ERROR; + } + + IPRT_DARWIN_RESTORE_EFL_AC(); + return VINF_SUCCESS; +} + + +/** + * This is a helper function to executes @a pfnWorker in the context of the + * kernel_task + * + * @returns IPRT status code - result from pfnWorker or dispatching error. + * @param pfnWorker The function to call. + * @param pArgs The arguments to pass to the function. + */ +static int rtR0MemObjDarwinDoInKernelTaskThread(thread_call_func_t pfnWorker, RTR0MEMOBJDARWINTHREADARGS *pArgs) +{ + pArgs->rc = VERR_IPE_UNINITIALIZED_STATUS; + pArgs->hEvent = NIL_RTSEMEVENTMULTI; + int rc = RTSemEventMultiCreate(&pArgs->hEvent); + if (RT_SUCCESS(rc)) + { + thread_call_t hCall = thread_call_allocate(pfnWorker, (void *)pArgs); + if (hCall) + { + boolean_t fRc = thread_call_enter(hCall); + AssertLogRel(fRc == FALSE); + + rc = RTSemEventMultiWaitEx(pArgs->hEvent, RTSEMWAIT_FLAGS_INDEFINITE | RTSEMWAIT_FLAGS_UNINTERRUPTIBLE, + RT_INDEFINITE_WAIT); + AssertLogRelRC(rc); + + rc = pArgs->rc; + thread_call_free(hCall); + } + else + rc = VERR_NO_MEMORY; + RTSemEventMultiDestroy(pArgs->hEvent); + } + return rc; +} + + +/** + * Signals result to thread waiting in rtR0MemObjDarwinDoInKernelTaskThread. + * + * @param pArgs The argument structure. + * @param rc The IPRT status code to signal. + */ +static void rtR0MemObjDarwinSignalThreadWaitinOnTask(RTR0MEMOBJDARWINTHREADARGS volatile *pArgs, int rc) +{ + if (ASMAtomicCmpXchgS32(&pArgs->rc, rc, VERR_IPE_UNINITIALIZED_STATUS)) + { + rc = RTSemEventMultiSignal(pArgs->hEvent); + AssertLogRelRC(rc); + } +} + + +/** + * Kernel memory alloc worker that uses inTaskWithPhysicalMask. + * + * @returns IPRT status code. + * @retval VERR_ADDRESS_TOO_BIG try another way. + * + * @param ppMem Where to return the memory object. + * @param cb The page aligned memory size. + * @param fExecutable Whether the mapping needs to be executable. + * @param fContiguous Whether the backing memory needs to be contiguous. + * @param PhysMask The mask for the backing memory (i.e. range). Use 0 if + * you don't care that much or is speculating. + * @param MaxPhysAddr The max address to verify the result against. Use + * UINT64_MAX if it doesn't matter. + * @param enmType The object type. + * @param uAlignment The allocation alignment (in bytes). + * @param pszTag Allocation tag used for statistics and such. + * @param fOnKernelThread Set if we're already on the kernel thread. + */ +static int rtR0MemObjNativeAllocWorker(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, + bool fExecutable, bool fContiguous, + mach_vm_address_t PhysMask, uint64_t MaxPhysAddr, + RTR0MEMOBJTYPE enmType, size_t uAlignment, const char *pszTag, bool fOnKernelThread) +{ + int rc; + + /* + * Because of process code signing properties leaking into kernel space in + * in XNU's vm_fault.c code, we have to defer allocations of exec memory to + * a thread running in the kernel_task to get consistent results here. + * + * Trouble strikes in vm_fault_enter() when cs_enforcement_enabled is determined + * to be true because current process has the CS_ENFORCEMENT flag, the page flag + * vmp_cs_validated is clear, and the protection mask includes VM_PROT_EXECUTE + * (pmap_cs_enforced does not apply to macOS it seems). This test seems to go + * back to 10.5, though I'm not sure whether it's enabled for macOS that early + * on. Only VM_PROT_EXECUTE is problematic for kernel memory, (though + * VM_PROT_WRITE on code signed pages is also problematic in theory). As long as + * kernel_task doesn't have CS_ENFORCEMENT enabled, we'll be fine switching to it. + */ + if (!fExecutable || fOnKernelThread) + { /* likely */ } + else + { + RTR0MEMOBJDARWINALLOCARGS Args; + Args.ppMem = ppMem; + Args.cb = cb; + Args.fExecutable = fExecutable; + Args.fContiguous = fContiguous; + Args.PhysMask = PhysMask; + Args.MaxPhysAddr = MaxPhysAddr; + Args.enmType = enmType; + Args.uAlignment = uAlignment; + Args.pszTag = pszTag; + return rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeAllockWorkerOnKernelThread, &Args.Core); + } + + /* + * Try inTaskWithPhysicalMask first, but since we don't quite trust that it + * actually respects the physical memory mask (10.5.x is certainly busted), + * we'll use rtR0MemObjNativeAllocCont as a fallback for dealing with that. + * + * The kIOMemoryKernelUserShared flag just forces the result to be page aligned. + * + * The kIOMemoryMapperNone flag is required since 10.8.2 (IOMMU changes?). + */ + + /* This is an old fudge from the snow leoard days: "Is it only on snow leopard? + Seen allocating memory for the VM structure, last page corrupted or + inaccessible." Made it only apply to snow leopard and older for now. */ + size_t cbFudged = cb; + if (version_major >= 11 /* 10 = 10.7.x = Lion. */) + { /* likely */ } + else + cbFudged += PAGE_SIZE; + + IOOptionBits fOptions = kIOMemoryKernelUserShared | kIODirectionInOut; + if (fContiguous) + { + fOptions |= kIOMemoryPhysicallyContiguous; + if ( version_major > 12 + || (version_major == 12 && version_minor >= 2) /* 10.8.2 = Mountain Kitten */ ) + fOptions |= kIOMemoryHostPhysicallyContiguous; /* (Just to make ourselves clear, in case the xnu code changes.) */ + } + if (version_major >= 12 /* 12 = 10.8.x = Mountain Kitten */) + fOptions |= kIOMemoryMapperNone; + +#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 && 0 /* enable when/if necessary */ + /* Paranoia: Don't misrepresent our intentions, we won't map kernel executable memory into ring-0. */ + if (fExecutable && version_major >= 11 /* 10.7.x = Lion, as below */) + { + fOptions &= ~kIOMemoryKernelUserShared; + if (uAlignment < PAGE_SIZE) + uAlignment = PAGE_SIZE; + } +#endif + + /* The public initWithPhysicalMask virtual method appeared in 10.7.0, in + versions 10.5.0 up to 10.7.0 it was private, and 10.4.8-10.5.0 it was + x86 only and didn't have the alignment parameter (slot was different too). */ + uint64_t uAlignmentActual = uAlignment; + IOBufferMemoryDescriptor *pMemDesc; +#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 + if (version_major >= 11 /* 11 = 10.7.x = Lion, could probably allow 10.5.0+ here if we really wanted to. */) + { + /* Starting with 10.6.x the physical mask is ignored if alignment is higher + than 1. The assumption seems to be that inTaskWithPhysicalMask() should + be used and the alignment inferred from the PhysMask argument. */ + if (MaxPhysAddr != UINT64_MAX) + { + Assert(RT_ALIGN_64(PhysMask, uAlignment) == PhysMask); + uAlignmentActual = 1; + } + + pMemDesc = new IOBufferMemoryDescriptor; + if (pMemDesc) + { + if (pMemDesc->initWithPhysicalMask(kernel_task, fOptions, cbFudged, uAlignmentActual, PhysMask)) + { /* likely */ } + else + { + pMemDesc->release(); + pMemDesc = NULL; + } + } + } + else +#endif + pMemDesc = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task, fOptions, cbFudged, PhysMask); + if (pMemDesc) + { + IOReturn IORet = pMemDesc->prepare(kIODirectionInOut); + if (IORet == kIOReturnSuccess) + { + void *pv = pMemDesc->getBytesNoCopy(0, cbFudged); + if (pv) + { + /* + * Check if it's all below 4GB. + */ + addr64_t AddrPrev = 0; + MaxPhysAddr &= ~(uint64_t)PAGE_OFFSET_MASK; + for (IOByteCount off = 0; off < cb; off += PAGE_SIZE) + { +#ifdef __LP64__ + addr64_t Addr = pMemDesc->getPhysicalSegment(off, NULL, kIOMemoryMapperNone); +#else + addr64_t Addr = pMemDesc->getPhysicalSegment64(off, NULL); +#endif + if ( Addr > MaxPhysAddr + || !Addr + || (Addr & PAGE_OFFSET_MASK) + || ( fContiguous + && !off + && Addr == AddrPrev + PAGE_SIZE)) + { + /* Buggy API, try allocate the memory another way. */ + pMemDesc->complete(); + pMemDesc->release(); + if (PhysMask) + { + kprintf("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x - buggy API!\n", + (size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions); + LogRel(("rtR0MemObjNativeAllocWorker: off=%zx Addr=%llx AddrPrev=%llx MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x - buggy API!\n", + (size_t)off, Addr, AddrPrev, MaxPhysAddr, PhysMask, fContiguous, fOptions)); + } + return VERR_ADDRESS_TOO_BIG; + } + AddrPrev = Addr; + } + + /* + * Check that it's aligned correctly. + */ + if ((uintptr_t)pv & (uAlignment - 1)) + { + pMemDesc->complete(); + pMemDesc->release(); + if (PhysMask) + { + kprintf("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%d fOptions=%#x) - buggy API!!\n", + pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions); + LogRel(("rtR0MemObjNativeAllocWorker: pv=%p uAlignment=%#zx (MaxPhysAddr=%llx PhysMas=%llx fContiguous=%RTbool fOptions=%#x) - buggy API!\n", + pv, uAlignment, MaxPhysAddr, PhysMask, fContiguous, fOptions)); + } + return VERR_NOT_SUPPORTED; + } + +#ifdef RT_STRICT + /* check that the memory is actually mapped. */ + //addr64_t Addr = pMemDesc->getPhysicalSegment64(0, NULL); + //printf("rtR0MemObjNativeAllocWorker: pv=%p %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr); + RTTHREADPREEMPTSTATE State = RTTHREADPREEMPTSTATE_INITIALIZER; + RTThreadPreemptDisable(&State); + rtR0MemObjDarwinTouchPages(pv, cb); + RTThreadPreemptRestore(&State); +#endif + + /* + * Create the IPRT memory object. + */ + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), enmType, pv, cb, pszTag); + if (pMemDarwin) + { + if (fOptions & kIOMemoryKernelUserShared) + pMemDarwin->Core.fFlags |= RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC; + else + pMemDarwin->Core.fFlags |= RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC; + if (fContiguous) + { +#ifdef __LP64__ + addr64_t PhysBase64 = pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone); +#else + addr64_t PhysBase64 = pMemDesc->getPhysicalSegment64(0, NULL); +#endif + RTHCPHYS PhysBase = PhysBase64; Assert(PhysBase == PhysBase64); + if (enmType == RTR0MEMOBJTYPE_CONT) + pMemDarwin->Core.u.Cont.Phys = PhysBase; + else if (enmType == RTR0MEMOBJTYPE_PHYS) + pMemDarwin->Core.u.Phys.PhysBase = PhysBase; + else + AssertMsgFailed(("enmType=%d\n", enmType)); + } + + if (fExecutable) + { + rc = rtR0MemObjNativeProtectWorker(&pMemDarwin->Core, 0, cb, + RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC); +#ifdef RT_STRICT + if (RT_SUCCESS(rc)) + { + /* check that the memory is actually mapped. */ + RTTHREADPREEMPTSTATE State2 = RTTHREADPREEMPTSTATE_INITIALIZER; + RTThreadPreemptDisable(&State2); + rtR0MemObjDarwinTouchPages(pv, cb); + RTThreadPreemptRestore(&State2); + } +#endif + /* Bug 6226: Ignore KERN_PROTECTION_FAILURE on Leopard and older. */ + if ( rc == VERR_PERMISSION_DENIED + && version_major <= 10 /* 10 = 10.6.x = Snow Leopard. */) + rc = VINF_SUCCESS; + } + else + rc = VINF_SUCCESS; + if (RT_SUCCESS(rc)) + { + pMemDarwin->pMemDesc = pMemDesc; + *ppMem = &pMemDarwin->Core; + return VINF_SUCCESS; + } + + rtR0MemObjDelete(&pMemDarwin->Core); + } + + if (enmType == RTR0MEMOBJTYPE_PHYS_NC) + rc = VERR_NO_PHYS_MEMORY; + else if (enmType == RTR0MEMOBJTYPE_LOW) + rc = VERR_NO_LOW_MEMORY; + else if (enmType == RTR0MEMOBJTYPE_CONT) + rc = VERR_NO_CONT_MEMORY; + else + rc = VERR_NO_MEMORY; + } + else + rc = VERR_MEMOBJ_INIT_FAILED; + + pMemDesc->complete(); + } + else + rc = RTErrConvertFromDarwinIO(IORet); + pMemDesc->release(); + } + else + rc = VERR_MEMOBJ_INIT_FAILED; + Assert(rc != VERR_ADDRESS_TOO_BIG); + return rc; +} + + +/** + * rtR0MemObjNativeAllocWorker kernel_task wrapper function. + */ +static void rtR0MemObjNativeAllockWorkerOnKernelThread(void *pvUser0, void *pvUser1) +{ + AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1); + RTR0MEMOBJDARWINALLOCARGS volatile *pArgs = (RTR0MEMOBJDARWINALLOCARGS volatile *)pvUser0; + int rc = rtR0MemObjNativeAllocWorker(pArgs->ppMem, pArgs->cb, pArgs->fExecutable, pArgs->fContiguous, pArgs->PhysMask, + pArgs->MaxPhysAddr, pArgs->enmType, pArgs->uAlignment, pArgs->pszTag, + true /*fOnKernelThread*/); + rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc); +} + + +DECLHIDDEN(int) rtR0MemObjNativeAllocPage(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag) +{ + IPRT_DARWIN_SAVE_EFL_AC(); + + int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, 0 /* PhysMask */, UINT64_MAX, + RTR0MEMOBJTYPE_PAGE, PAGE_SIZE, pszTag, false /*fOnKernelThread*/); + + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +DECLHIDDEN(int) rtR0MemObjNativeAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags, + const char *pszTag) +{ + return rtR0MemObjFallbackAllocLarge(ppMem, cb, cbLargePage, fFlags, pszTag); +} + + +DECLHIDDEN(int) rtR0MemObjNativeAllocLow(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag) +{ + IPRT_DARWIN_SAVE_EFL_AC(); + + /* + * Try IOMallocPhysical/IOMallocAligned first. + * Then try optimistically without a physical address mask, which will always + * end up using IOMallocAligned. + * + * (See bug comment in the worker and IOBufferMemoryDescriptor::initWithPhysicalMask.) + */ + int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, ~(uint32_t)PAGE_OFFSET_MASK, + _4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, pszTag, false /*fOnKernelThread*/); + if (rc == VERR_ADDRESS_TOO_BIG) + rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, false /* fContiguous */, 0 /* PhysMask */, + _4G - PAGE_SIZE, RTR0MEMOBJTYPE_LOW, PAGE_SIZE, pszTag, false /*fOnKernelThread*/); + + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +DECLHIDDEN(int) rtR0MemObjNativeAllocCont(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, bool fExecutable, const char *pszTag) +{ + IPRT_DARWIN_SAVE_EFL_AC(); + + int rc = rtR0MemObjNativeAllocWorker(ppMem, cb, fExecutable, true /* fContiguous */, + ~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE, + RTR0MEMOBJTYPE_CONT, PAGE_SIZE, pszTag, false /*fOnKernelThread*/); + + /* + * Workaround for bogus IOKernelAllocateContiguous behavior, just in case. + * cb <= PAGE_SIZE allocations take a different path, using a different allocator. + */ + if (RT_FAILURE(rc) && cb <= PAGE_SIZE) + rc = rtR0MemObjNativeAllocWorker(ppMem, cb + PAGE_SIZE, fExecutable, true /* fContiguous */, + ~(uint32_t)PAGE_OFFSET_MASK, _4G - PAGE_SIZE, + RTR0MEMOBJTYPE_CONT, PAGE_SIZE, pszTag, false /*fOnKernelThread*/); + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +DECLHIDDEN(int) rtR0MemObjNativeAllocPhys(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment, + const char *pszTag) +{ + if (uAlignment != PAGE_SIZE) + { + /* See rtR0MemObjNativeAllocWorker: */ + if (version_major < 9 /* 9 = 10.5.x = Snow Leopard */) + return VERR_NOT_SUPPORTED; + } + + IPRT_DARWIN_SAVE_EFL_AC(); + + /* + * Translate the PhysHighest address into a mask. + */ + int rc; + if (PhysHighest == NIL_RTHCPHYS) + rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable */, true /* fContiguous */, + uAlignment <= PAGE_SIZE ? 0 : ~(mach_vm_address_t)(uAlignment - 1) /* PhysMask*/, + UINT64_MAX, RTR0MEMOBJTYPE_PHYS, uAlignment, pszTag, false /*fOnKernelThread*/); + else + { + mach_vm_address_t PhysMask = 0; + PhysMask = ~(mach_vm_address_t)0; + while (PhysMask > (PhysHighest | PAGE_OFFSET_MASK)) + PhysMask >>= 1; + AssertReturn(PhysMask + 1 <= cb, VERR_INVALID_PARAMETER); + PhysMask &= ~(mach_vm_address_t)(uAlignment - 1); + + rc = rtR0MemObjNativeAllocWorker(ppMem, cb, false /* fExecutable */, true /* fContiguous */, + PhysMask, PhysHighest, + RTR0MEMOBJTYPE_PHYS, uAlignment, pszTag, false /*fOnKernelThread*/); + } + + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +DECLHIDDEN(int) rtR0MemObjNativeAllocPhysNC(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, RTHCPHYS PhysHighest, const char *pszTag) +{ + /** @todo rtR0MemObjNativeAllocPhys / darwin. + * This might be a bit problematic and may very well require having to create our own + * object which we populate with pages but without mapping it into any address space. + * Estimate is 2-3 days. + */ + RT_NOREF(ppMem, cb, PhysHighest, pszTag); + return VERR_NOT_SUPPORTED; +} + + +DECLHIDDEN(int) rtR0MemObjNativeEnterPhys(PPRTR0MEMOBJINTERNAL ppMem, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy, + const char *pszTag) +{ + AssertReturn(uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE, VERR_NOT_SUPPORTED); + IPRT_DARWIN_SAVE_EFL_AC(); + + /* + * Create a descriptor for it (the validation is always true on intel macs, but + * as it doesn't harm us keep it in). + */ + int rc = VERR_ADDRESS_TOO_BIG; + IOAddressRange aRanges[1] = { { Phys, cb } }; + if ( aRanges[0].address == Phys + && aRanges[0].length == cb) + { + IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRanges(&aRanges[0], RT_ELEMENTS(aRanges), + kIODirectionInOut, NULL /*task*/); + if (pMemDesc) + { +#ifdef __LP64__ + Assert(Phys == pMemDesc->getPhysicalSegment(0, NULL, kIOMemoryMapperNone)); +#else + Assert(Phys == pMemDesc->getPhysicalSegment64(0, NULL)); +#endif + + /* + * Create the IPRT memory object. + */ + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_PHYS, + NULL, cb, pszTag); + if (pMemDarwin) + { + pMemDarwin->Core.u.Phys.PhysBase = Phys; + pMemDarwin->Core.u.Phys.fAllocated = false; + pMemDarwin->Core.u.Phys.uCachePolicy = uCachePolicy; + pMemDarwin->pMemDesc = pMemDesc; + *ppMem = &pMemDarwin->Core; + IPRT_DARWIN_RESTORE_EFL_AC(); + return VINF_SUCCESS; + } + + rc = VERR_NO_MEMORY; + pMemDesc->release(); + } + else + rc = VERR_MEMOBJ_INIT_FAILED; + } + else + AssertMsgFailed(("%#llx %llx\n", (unsigned long long)Phys, (unsigned long long)cb)); + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +/** + * Internal worker for locking down pages. + * + * @return IPRT status code. + * + * @param ppMem Where to store the memory object pointer. + * @param pv First page. + * @param cb Number of bytes. + * @param fAccess The desired access, a combination of RTMEM_PROT_READ + * and RTMEM_PROT_WRITE. + * @param Task The task \a pv and \a cb refers to. + * @param pszTag Allocation tag used for statistics and such. + */ +static int rtR0MemObjNativeLock(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, task_t Task, + const char *pszTag) +{ + IPRT_DARWIN_SAVE_EFL_AC(); + NOREF(fAccess); +#ifdef USE_VM_MAP_WIRE + vm_map_t Map = get_task_map(Task); + Assert(Map); + + /* + * First try lock the memory. + */ + int rc = VERR_LOCK_FAILED; + kern_return_t kr = vm_map_wire(get_task_map(Task), + (vm_map_offset_t)pv, + (vm_map_offset_t)pv + cb, + VM_PROT_DEFAULT, + 0 /* not user */); + if (kr == KERN_SUCCESS) + { + /* + * Create the IPRT memory object. + */ + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK, pv, cb, pszTag); + if (pMemDarwin) + { + pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task; + *ppMem = &pMemDarwin->Core; + + IPRT_DARWIN_RESTORE_EFL_AC(); + return VINF_SUCCESS; + } + + kr = vm_map_unwire(get_task_map(Task), (vm_map_offset_t)pv, (vm_map_offset_t)pv + cb, 0 /* not user */); + Assert(kr == KERN_SUCCESS); + rc = VERR_NO_MEMORY; + } + +#else + + /* + * Create a descriptor and try lock it (prepare). + */ + int rc = VERR_MEMOBJ_INIT_FAILED; + IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withAddressRange((vm_address_t)pv, cb, kIODirectionInOut, Task); + if (pMemDesc) + { + IOReturn IORet = pMemDesc->prepare(kIODirectionInOut); + if (IORet == kIOReturnSuccess) + { + /* + * Create the IPRT memory object. + */ + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_LOCK, + pv, cb, pszTag); + if (pMemDarwin) + { + pMemDarwin->Core.u.Lock.R0Process = (RTR0PROCESS)Task; + pMemDarwin->pMemDesc = pMemDesc; + *ppMem = &pMemDarwin->Core; + + IPRT_DARWIN_RESTORE_EFL_AC(); + return VINF_SUCCESS; + } + + pMemDesc->complete(); + rc = VERR_NO_MEMORY; + } + else + rc = VERR_LOCK_FAILED; + pMemDesc->release(); + } +#endif + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +DECLHIDDEN(int) rtR0MemObjNativeLockUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, + RTR0PROCESS R0Process, const char *pszTag) +{ + return rtR0MemObjNativeLock(ppMem, (void *)R3Ptr, cb, fAccess, (task_t)R0Process, pszTag); +} + + +DECLHIDDEN(int) rtR0MemObjNativeLockKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pv, size_t cb, uint32_t fAccess, const char *pszTag) +{ + return rtR0MemObjNativeLock(ppMem, pv, cb, fAccess, kernel_task, pszTag); +} + + +DECLHIDDEN(int) rtR0MemObjNativeReserveKernel(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment, + const char *pszTag) +{ + RT_NOREF(ppMem, pvFixed, cb, uAlignment, pszTag); + return VERR_NOT_SUPPORTED; +} + + +DECLHIDDEN(int) rtR0MemObjNativeReserveUser(PPRTR0MEMOBJINTERNAL ppMem, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, + RTR0PROCESS R0Process, const char *pszTag) +{ + RT_NOREF(ppMem, R3PtrFixed, cb, uAlignment, R0Process, pszTag); + return VERR_NOT_SUPPORTED; +} + + +DECLHIDDEN(int) rtR0MemObjNativeMapKernel(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, void *pvFixed, size_t uAlignment, + unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag) +{ + RT_NOREF(fProt); + AssertReturn(pvFixed == (void *)-1, VERR_NOT_SUPPORTED); + + /* + * Check that the specified alignment is supported. + */ + if (uAlignment > PAGE_SIZE) + return VERR_NOT_SUPPORTED; + Assert(!offSub || cbSub); + + IPRT_DARWIN_SAVE_EFL_AC(); + + /* + * Must have a memory descriptor that we can map. + */ + int rc = VERR_INVALID_PARAMETER; + PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap; + if (pMemToMapDarwin->pMemDesc) + { + /* The kIOMapPrefault option was added in 10.10.0; causes PTEs to be populated with + INTEL_PTE_WIRED to be set, just like we desire (see further down). However, till + 10.13.0 it was not available for use on kernel mappings. Oh, fudge. */ +#if MAC_OS_X_VERSION_MIN_REQUIRED >= 1050 + static uint32_t volatile s_fOptions = UINT32_MAX; + uint32_t fOptions = s_fOptions; + if (RT_UNLIKELY(fOptions == UINT32_MAX)) + s_fOptions = fOptions = version_major >= 17 ? 0x10000000 /*kIOMapPrefault*/ : 0; /* Since 10.13.0 (High Sierra). */ + + IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask(kernel_task, + 0, + kIOMapAnywhere | kIOMapDefaultCache | fOptions, + offSub, + cbSub); +#else + IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map(kernel_task, + 0, + kIOMapAnywhere | kIOMapDefaultCache, + offSub, + cbSub); +#endif + if (pMemMap) + { + IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress(); + void *pv = (void *)(uintptr_t)VirtAddr; + if ((uintptr_t)pv == VirtAddr && pv != NULL) + { +//#ifdef __LP64__ +// addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone); +//#else +// addr64_t Addr = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL); +//#endif +// MY_PRINTF("pv=%p: %8llx %8llx\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr); + +// /* +// * Explicitly lock it so that we're sure it is present and that +// * its PTEs cannot be recycled. +// * Note! withAddressRange() doesn't work as it adds kIOMemoryTypeVirtual64 +// * to the options which causes prepare() to not wire the pages. +// * This is probably a bug. +// */ +// IOAddressRange Range = { (mach_vm_address_t)pv, cbSub }; +// IOMemoryDescriptor *pMemDesc = IOMemoryDescriptor::withOptions(&Range, +// 1 /* count */, +// 0 /* offset */, +// kernel_task, +// kIODirectionInOut | kIOMemoryTypeVirtual, +// kIOMapperSystem); +// if (pMemDesc) +// { +// IOReturn IORet = pMemDesc->prepare(kIODirectionInOut); +// if (IORet == kIOReturnSuccess) +// { + /* HACK ALERT! On kernels older than 10.10 (xnu version 14), we need to fault in + the pages here so they can safely be accessed from inside simple + locks and when preemption is disabled (no page-ins allowed). + Note! This touching does not cause INTEL_PTE_WIRED (bit 10) to be set as we go + thru general #PF and vm_fault doesn't figure it should be wired or something. */ + rtR0MemObjDarwinTouchPages(pv, cbSub ? cbSub : pMemToMap->cb); + /** @todo First, the memory should've been mapped by now, and second, it + * should have the wired attribute in the PTE (bit 10). Neither seems to + * be the case. The disabled locking code doesn't make any difference, + * which is extremely odd, and breaks rtR0MemObjNativeGetPagePhysAddr + * (getPhysicalSegment64 -> 64 for the lock descriptor. */ +//#ifdef __LP64__ +// addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment(offSub, NULL, kIOMemoryMapperNone); +//#else +// addr64_t Addr2 = pMemToMapDarwin->pMemDesc->getPhysicalSegment64(offSub, NULL); +//#endif +// MY_PRINTF("pv=%p: %8llx %8llx (%d)\n", pv, rtR0MemObjDarwinGetPTE(pv), Addr2, 2); + + /* + * Create the IPRT memory object. + */ + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING, + pv, cbSub ? cbSub : pMemToMap->cb, pszTag); + if (pMemDarwin) + { + pMemDarwin->Core.u.Mapping.R0Process = NIL_RTR0PROCESS; + pMemDarwin->pMemMap = pMemMap; +// pMemDarwin->pMemDesc = pMemDesc; + *ppMem = &pMemDarwin->Core; + + IPRT_DARWIN_RESTORE_EFL_AC(); + return VINF_SUCCESS; + } + +// pMemDesc->complete(); +// rc = VERR_NO_MEMORY; +// } +// else +// rc = RTErrConvertFromDarwinIO(IORet); +// pMemDesc->release(); +// } +// else +// rc = VERR_MEMOBJ_INIT_FAILED; + } + else if (pv) + rc = VERR_ADDRESS_TOO_BIG; + else + rc = VERR_MAP_FAILED; + pMemMap->release(); + } + else + rc = VERR_MAP_FAILED; + } + + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +DECLHIDDEN(int) rtR0MemObjNativeMapUser(PPRTR0MEMOBJINTERNAL ppMem, RTR0MEMOBJ pMemToMap, RTR3PTR R3PtrFixed, size_t uAlignment, + unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag) +{ + RT_NOREF(fProt); + + /* + * Check for unsupported things. + */ + AssertReturn(R3PtrFixed == (RTR3PTR)-1, VERR_NOT_SUPPORTED); + if (uAlignment > PAGE_SIZE) + return VERR_NOT_SUPPORTED; + Assert(!offSub || cbSub); + + IPRT_DARWIN_SAVE_EFL_AC(); + + /* + * Must have a memory descriptor. + */ + int rc = VERR_INVALID_PARAMETER; + PRTR0MEMOBJDARWIN pMemToMapDarwin = (PRTR0MEMOBJDARWIN)pMemToMap; + if (pMemToMapDarwin->pMemDesc) + { +#if MAC_OS_X_VERSION_MIN_REQUIRED >= 101000 /* The kIOMapPrefault option was added in 10.10.0. */ + IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process, + 0, + kIOMapAnywhere | kIOMapDefaultCache | kIOMapPrefault, + offSub, + cbSub); +#elif MAC_OS_X_VERSION_MIN_REQUIRED >= 1050 + static uint32_t volatile s_fOptions = UINT32_MAX; + uint32_t fOptions = s_fOptions; + if (RT_UNLIKELY(fOptions == UINT32_MAX)) + s_fOptions = fOptions = version_major >= 14 ? 0x10000000 /*kIOMapPrefault*/ : 0; /* Since 10.10.0. */ + IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->createMappingInTask((task_t)R0Process, + 0, + kIOMapAnywhere | kIOMapDefaultCache | fOptions, + offSub, + cbSub); +#else + IOMemoryMap *pMemMap = pMemToMapDarwin->pMemDesc->map((task_t)R0Process, + 0, + kIOMapAnywhere | kIOMapDefaultCache, + offSub, + cbSub); +#endif + if (pMemMap) + { + IOVirtualAddress VirtAddr = pMemMap->getVirtualAddress(); + void *pv = (void *)(uintptr_t)VirtAddr; + if ((uintptr_t)pv == VirtAddr && pv != NULL) + { + /* + * Create the IPRT memory object. + */ + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)rtR0MemObjNew(sizeof(*pMemDarwin), RTR0MEMOBJTYPE_MAPPING, + pv, cbSub ? cbSub : pMemToMap->cb, pszTag); + if (pMemDarwin) + { + pMemDarwin->Core.u.Mapping.R0Process = R0Process; + pMemDarwin->pMemMap = pMemMap; + *ppMem = &pMemDarwin->Core; + + IPRT_DARWIN_RESTORE_EFL_AC(); + return VINF_SUCCESS; + } + + rc = VERR_NO_MEMORY; + } + else if (pv) + rc = VERR_ADDRESS_TOO_BIG; + else + rc = VERR_MAP_FAILED; + pMemMap->release(); + } + else + rc = VERR_MAP_FAILED; + } + + IPRT_DARWIN_RESTORE_EFL_AC(); + return rc; +} + + +/** + * Worker for rtR0MemObjNativeProtect that's typically called in a different + * context. + */ +static int rtR0MemObjNativeProtectWorker(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt) +{ + IPRT_DARWIN_SAVE_EFL_AC(); + + /* Get the map for the object. */ + vm_map_t pVmMap = rtR0MemObjDarwinGetMap(pMem); + if (!pVmMap) + { + IPRT_DARWIN_RESTORE_EFL_AC(); + return VERR_NOT_SUPPORTED; + } + + /* + * Convert the protection. + */ + vm_prot_t fMachProt; + switch (fProt) + { + case RTMEM_PROT_NONE: + fMachProt = VM_PROT_NONE; + break; + case RTMEM_PROT_READ: + fMachProt = VM_PROT_READ; + break; + case RTMEM_PROT_READ | RTMEM_PROT_WRITE: + fMachProt = VM_PROT_READ | VM_PROT_WRITE; + break; + case RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC: + fMachProt = VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE; + break; + case RTMEM_PROT_WRITE: + fMachProt = VM_PROT_WRITE | VM_PROT_READ; /* never write-only */ + break; + case RTMEM_PROT_WRITE | RTMEM_PROT_EXEC: + fMachProt = VM_PROT_WRITE | VM_PROT_EXECUTE | VM_PROT_READ; /* never write-only or execute-only */ + break; + case RTMEM_PROT_EXEC: + fMachProt = VM_PROT_EXECUTE | VM_PROT_READ; /* never execute-only */ + break; + default: + AssertFailedReturn(VERR_INVALID_PARAMETER); + } + + /* + * Do the job. + */ + vm_offset_t Start = (uintptr_t)pMem->pv + offSub; + kern_return_t krc = vm_protect(pVmMap, + Start, + cbSub, + false, + fMachProt); + if (krc != KERN_SUCCESS) + { + static int s_cComplaints = 0; + if (s_cComplaints < 10) + { + s_cComplaints++; + printf("rtR0MemObjNativeProtect: vm_protect(%p,%p,%p,false,%#x) -> %d\n", + (void *)pVmMap, (void *)Start, (void *)cbSub, fMachProt, krc); + + kern_return_t krc2; + vm_offset_t pvReal = Start; + vm_size_t cbReal = 0; + mach_msg_type_number_t cInfo = VM_REGION_BASIC_INFO_COUNT; + struct vm_region_basic_info Info; + RT_ZERO(Info); + krc2 = vm_region(pVmMap, &pvReal, &cbReal, VM_REGION_BASIC_INFO, (vm_region_info_t)&Info, &cInfo, NULL); + printf("rtR0MemObjNativeProtect: basic info - krc2=%d pv=%p cb=%p prot=%#x max=%#x inh=%#x shr=%d rvd=%d off=%#x behavior=%#x wired=%#x\n", + krc2, (void *)pvReal, (void *)cbReal, Info.protection, Info.max_protection, Info.inheritance, + Info.shared, Info.reserved, Info.offset, Info.behavior, Info.user_wired_count); + } + IPRT_DARWIN_RESTORE_EFL_AC(); + return RTErrConvertFromDarwinKern(krc); + } + + /* + * Touch the pages if they should be writable afterwards and accessible + * from code which should never fault. vm_protect() may leave pages + * temporarily write protected, possibly due to pmap no-upgrade rules? + * + * This is the same trick (or HACK ALERT if you like) as applied in + * rtR0MemObjNativeMapKernel. + */ + if ( pMem->enmType != RTR0MEMOBJTYPE_MAPPING + || pMem->u.Mapping.R0Process == NIL_RTR0PROCESS) + { + if (fProt & RTMEM_PROT_WRITE) + rtR0MemObjDarwinTouchPages((void *)Start, cbSub); + /* + * Sniff (read) read-only pages too, just to be sure. + */ + else if (fProt & (RTMEM_PROT_READ | RTMEM_PROT_EXEC)) + rtR0MemObjDarwinSniffPages((void const *)Start, cbSub); + } + + IPRT_DARWIN_RESTORE_EFL_AC(); + return VINF_SUCCESS; +} + + +/** + * rtR0MemObjNativeProtect kernel_task wrapper function. + */ +static void rtR0MemObjNativeProtectWorkerOnKernelThread(void *pvUser0, void *pvUser1) +{ + AssertPtr(pvUser0); Assert(pvUser1 == NULL); NOREF(pvUser1); + RTR0MEMOBJDARWINPROTECTARGS *pArgs = (RTR0MEMOBJDARWINPROTECTARGS *)pvUser0; + int rc = rtR0MemObjNativeProtectWorker(pArgs->pMem, pArgs->offSub, pArgs->cbSub, pArgs->fProt); + rtR0MemObjDarwinSignalThreadWaitinOnTask(&pArgs->Core, rc); +} + + +DECLHIDDEN(int) rtR0MemObjNativeProtect(PRTR0MEMOBJINTERNAL pMem, size_t offSub, size_t cbSub, uint32_t fProt) +{ + /* + * The code won't work right because process codesigning properties leaks + * into kernel_map memory management. So, if the user process we're running + * in has CS restrictions active, we cannot play around with the EXEC + * protection because some vm_fault.c think we're modifying the process map + * or something. + */ + int rc; + if (rtR0MemObjDarwinGetMap(pMem) == kernel_map) + { + RTR0MEMOBJDARWINPROTECTARGS Args; + Args.pMem = pMem; + Args.offSub = offSub; + Args.cbSub = cbSub; + Args.fProt = fProt; + rc = rtR0MemObjDarwinDoInKernelTaskThread(rtR0MemObjNativeProtectWorkerOnKernelThread, &Args.Core); + } + else + rc = rtR0MemObjNativeProtectWorker(pMem, offSub, cbSub, fProt); + return rc; +} + + +DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage) +{ + RTHCPHYS PhysAddr; + PRTR0MEMOBJDARWIN pMemDarwin = (PRTR0MEMOBJDARWIN)pMem; + IPRT_DARWIN_SAVE_EFL_AC(); + +#ifdef USE_VM_MAP_WIRE + /* + * Locked memory doesn't have a memory descriptor and + * needs to be handled differently. + */ + if (pMemDarwin->Core.enmType == RTR0MEMOBJTYPE_LOCK) + { + ppnum_t PgNo; + if (pMemDarwin->Core.u.Lock.R0Process == NIL_RTR0PROCESS) + PgNo = pmap_find_phys(kernel_pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE); + else + { + /* + * From what I can tell, Apple seems to have locked up the all the + * available interfaces that could help us obtain the pmap_t of a task + * or vm_map_t. + + * So, we'll have to figure out where in the vm_map_t structure it is + * and read it our selves. ASSUMING that kernel_pmap is pointed to by + * kernel_map->pmap, we scan kernel_map to locate the structure offset. + * Not nice, but it will hopefully do the job in a reliable manner... + * + * (get_task_pmap, get_map_pmap or vm_map_pmap is what we really need btw.) + */ + static int s_offPmap = -1; + if (RT_UNLIKELY(s_offPmap == -1)) + { + pmap_t const *p = (pmap_t *)kernel_map; + pmap_t const * const pEnd = p + 64; + for (; p < pEnd; p++) + if (*p == kernel_pmap) + { + s_offPmap = (uintptr_t)p - (uintptr_t)kernel_map; + break; + } + AssertReturn(s_offPmap >= 0, NIL_RTHCPHYS); + } + pmap_t Pmap = *(pmap_t *)((uintptr_t)get_task_map((task_t)pMemDarwin->Core.u.Lock.R0Process) + s_offPmap); + PgNo = pmap_find_phys(Pmap, (uintptr_t)pMemDarwin->Core.pv + iPage * PAGE_SIZE); + } + + IPRT_DARWIN_RESTORE_EFL_AC(); + AssertReturn(PgNo, NIL_RTHCPHYS); + PhysAddr = (RTHCPHYS)PgNo << PAGE_SHIFT; + Assert((PhysAddr >> PAGE_SHIFT) == PgNo); + } + else +#endif /* USE_VM_MAP_WIRE */ + { + /* + * Get the memory descriptor. + */ + IOMemoryDescriptor *pMemDesc = pMemDarwin->pMemDesc; + if (!pMemDesc) + pMemDesc = pMemDarwin->pMemMap->getMemoryDescriptor(); + AssertReturn(pMemDesc, NIL_RTHCPHYS); + + /* + * If we've got a memory descriptor, use getPhysicalSegment64(). + */ +#ifdef __LP64__ + addr64_t Addr = pMemDesc->getPhysicalSegment(iPage * PAGE_SIZE, NULL, kIOMemoryMapperNone); +#else + addr64_t Addr = pMemDesc->getPhysicalSegment64(iPage * PAGE_SIZE, NULL); +#endif + IPRT_DARWIN_RESTORE_EFL_AC(); + AssertMsgReturn(Addr, ("iPage=%u\n", iPage), NIL_RTHCPHYS); + PhysAddr = Addr; + AssertMsgReturn(PhysAddr == Addr, ("PhysAddr=%RHp Addr=%RX64\n", PhysAddr, (uint64_t)Addr), NIL_RTHCPHYS); + } + + return PhysAddr; +} + |