/* $Id: memobj-r0drv.cpp $ */ /** @file * IPRT - Ring-0 Memory Objects, Common Code. */ /* * Copyright (C) 2006-2022 Oracle and/or its affiliates. * * This file is part of VirtualBox base platform packages, as * available from https://www.virtualbox.org. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation, in version 3 of the * License. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, see . * * The contents of this file may alternatively be used under the terms * of the Common Development and Distribution License Version 1.0 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included * in the VirtualBox distribution, in which case the provisions of the * CDDL are applicable instead of those of the GPL. * * You may elect to license modified versions of this file under the * terms and conditions of either the GPL or the CDDL or both. * * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 */ /********************************************************************************************************************************* * Header Files * *********************************************************************************************************************************/ #define LOG_GROUP RTLOGGROUP_DEFAULT /// @todo RTLOGGROUP_MEM #define RTMEM_NO_WRAP_TO_EF_APIS /* circular dependency otherwise. */ #include #include "internal/iprt.h" #include #include #include #include #include #include #include #include #include #include "internal/memobj.h" /** * Internal function for allocating a new memory object. * * @returns The allocated and initialized handle. * @param cbSelf The size of the memory object handle. 0 mean default size. * @param enmType The memory object type. * @param pv The memory object mapping. * @param cb The size of the memory object. * @param pszTag The tag string. */ DECLHIDDEN(PRTR0MEMOBJINTERNAL) rtR0MemObjNew(size_t cbSelf, RTR0MEMOBJTYPE enmType, void *pv, size_t cb, const char *pszTag) { PRTR0MEMOBJINTERNAL pNew; /* validate the size */ if (!cbSelf) cbSelf = sizeof(*pNew); Assert(cbSelf >= sizeof(*pNew)); Assert(cbSelf == (uint32_t)cbSelf); AssertMsg(RT_ALIGN_Z(cb, PAGE_SIZE) == cb, ("%#zx\n", cb)); /* * Allocate and initialize the object. */ pNew = (PRTR0MEMOBJINTERNAL)RTMemAllocZ(cbSelf); if (pNew) { pNew->u32Magic = RTR0MEMOBJ_MAGIC; pNew->cbSelf = (uint32_t)cbSelf; pNew->enmType = enmType; pNew->fFlags = 0; pNew->cb = cb; pNew->pv = pv; #ifdef DEBUG pNew->pszTag = pszTag; #else RT_NOREF_PV(pszTag); #endif } return pNew; } /** * Deletes an incomplete memory object. * * This is for cleaning up after failures during object creation. * * @param pMem The incomplete memory object to delete. */ DECLHIDDEN(void) rtR0MemObjDelete(PRTR0MEMOBJINTERNAL pMem) { if (pMem) { ASMAtomicUoWriteU32(&pMem->u32Magic, ~RTR0MEMOBJ_MAGIC); pMem->enmType = RTR0MEMOBJTYPE_END; RTMemFree(pMem); } } /** * Links a mapping object to a primary object. * * @returns IPRT status code. * @retval VINF_SUCCESS on success. * @retval VINF_NO_MEMORY if we couldn't expand the mapping array of the parent. * @param pParent The parent (primary) memory object. * @param pChild The child (mapping) memory object. */ static int rtR0MemObjLink(PRTR0MEMOBJINTERNAL pParent, PRTR0MEMOBJINTERNAL pChild) { uint32_t i; /* sanity */ Assert(rtR0MemObjIsMapping(pChild)); Assert(!rtR0MemObjIsMapping(pParent)); /* expand the array? */ i = pParent->uRel.Parent.cMappings; if (i >= pParent->uRel.Parent.cMappingsAllocated) { void *pv = RTMemRealloc(pParent->uRel.Parent.papMappings, (i + 32) * sizeof(pParent->uRel.Parent.papMappings[0])); if (!pv) return VERR_NO_MEMORY; pParent->uRel.Parent.papMappings = (PPRTR0MEMOBJINTERNAL)pv; pParent->uRel.Parent.cMappingsAllocated = i + 32; Assert(i == pParent->uRel.Parent.cMappings); } /* do the linking. */ pParent->uRel.Parent.papMappings[i] = pChild; pParent->uRel.Parent.cMappings++; pChild->uRel.Child.pParent = pParent; return VINF_SUCCESS; } /** * Checks if this is mapping or not. * * @returns true if it's a mapping, otherwise false. * @param MemObj The ring-0 memory object handle. */ RTR0DECL(bool) RTR0MemObjIsMapping(RTR0MEMOBJ MemObj) { /* Validate the object handle. */ PRTR0MEMOBJINTERNAL pMem; AssertPtrReturn(MemObj, false); pMem = (PRTR0MEMOBJINTERNAL)MemObj; AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), false); AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), false); /* hand it on to the inlined worker. */ return rtR0MemObjIsMapping(pMem); } RT_EXPORT_SYMBOL(RTR0MemObjIsMapping); /** * Gets the address of a ring-0 memory object. * * @returns The address of the memory object. * @returns NULL if the handle is invalid (asserts in strict builds) or if there isn't any mapping. * @param MemObj The ring-0 memory object handle. */ RTR0DECL(void *) RTR0MemObjAddress(RTR0MEMOBJ MemObj) { /* Validate the object handle. */ PRTR0MEMOBJINTERNAL pMem; if (RT_UNLIKELY(MemObj == NIL_RTR0MEMOBJ)) return NULL; AssertPtrReturn(MemObj, NULL); pMem = (PRTR0MEMOBJINTERNAL)MemObj; AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), NULL); AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), NULL); /* return the mapping address. */ return pMem->pv; } RT_EXPORT_SYMBOL(RTR0MemObjAddress); /** * Gets the ring-3 address of a ring-0 memory object. * * This only applies to ring-0 memory object with ring-3 mappings of some kind, i.e. * locked user memory, reserved user address space and user mappings. This API should * not be used on any other objects. * * @returns The address of the memory object. * @returns NIL_RTR3PTR if the handle is invalid or if it's not an object with a ring-3 mapping. * Strict builds will assert in both cases. * @param MemObj The ring-0 memory object handle. */ RTR0DECL(RTR3PTR) RTR0MemObjAddressR3(RTR0MEMOBJ MemObj) { PRTR0MEMOBJINTERNAL pMem; /* Validate the object handle. */ if (RT_UNLIKELY(MemObj == NIL_RTR0MEMOBJ)) return NIL_RTR3PTR; AssertPtrReturn(MemObj, NIL_RTR3PTR); pMem = (PRTR0MEMOBJINTERNAL)MemObj; AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), NIL_RTR3PTR); AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), NIL_RTR3PTR); if (RT_UNLIKELY( ( pMem->enmType != RTR0MEMOBJTYPE_MAPPING || pMem->u.Mapping.R0Process == NIL_RTR0PROCESS) && ( pMem->enmType != RTR0MEMOBJTYPE_LOCK || pMem->u.Lock.R0Process == NIL_RTR0PROCESS) && ( pMem->enmType != RTR0MEMOBJTYPE_PHYS_NC || pMem->u.Lock.R0Process == NIL_RTR0PROCESS) && ( pMem->enmType != RTR0MEMOBJTYPE_RES_VIRT || pMem->u.ResVirt.R0Process == NIL_RTR0PROCESS))) return NIL_RTR3PTR; /* return the mapping address. */ return (RTR3PTR)pMem->pv; } RT_EXPORT_SYMBOL(RTR0MemObjAddressR3); /** * Gets the size of a ring-0 memory object. * * The returned value may differ from the one specified to the API creating the * object because of alignment adjustments. The minimal alignment currently * employed by any API is PAGE_SIZE, so the result can safely be shifted by * PAGE_SHIFT to calculate a page count. * * @returns The object size. * @returns 0 if the handle is invalid (asserts in strict builds) or if there isn't any mapping. * @param MemObj The ring-0 memory object handle. */ RTR0DECL(size_t) RTR0MemObjSize(RTR0MEMOBJ MemObj) { PRTR0MEMOBJINTERNAL pMem; /* Validate the object handle. */ if (RT_UNLIKELY(MemObj == NIL_RTR0MEMOBJ)) return 0; AssertPtrReturn(MemObj, 0); pMem = (PRTR0MEMOBJINTERNAL)MemObj; AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), 0); AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), 0); AssertMsg(RT_ALIGN_Z(pMem->cb, PAGE_SIZE) == pMem->cb, ("%#zx\n", pMem->cb)); /* return the size. */ return pMem->cb; } RT_EXPORT_SYMBOL(RTR0MemObjSize); /** * Get the physical address of an page in the memory object. * * @returns The physical address. * @returns NIL_RTHCPHYS if the object doesn't contain fixed physical pages. * @returns NIL_RTHCPHYS if the iPage is out of range. * @returns NIL_RTHCPHYS if the object handle isn't valid. * @param MemObj The ring-0 memory object handle. * @param iPage The page number within the object. */ /* Work around gcc bug 55940 */ #if defined(__GNUC__) && defined(RT_ARCH_X86) && (__GNUC__ * 100 + __GNUC_MINOR__) == 407 __attribute__((__optimize__ ("no-shrink-wrap"))) #endif RTR0DECL(RTHCPHYS) RTR0MemObjGetPagePhysAddr(RTR0MEMOBJ MemObj, size_t iPage) { /* Validate the object handle. */ PRTR0MEMOBJINTERNAL pMem; size_t cPages; AssertPtrReturn(MemObj, NIL_RTHCPHYS); pMem = (PRTR0MEMOBJINTERNAL)MemObj; AssertReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, NIL_RTHCPHYS); AssertReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, NIL_RTHCPHYS); AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), NIL_RTHCPHYS); AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), NIL_RTHCPHYS); cPages = (pMem->cb >> PAGE_SHIFT); if (iPage >= cPages) { /* permit: while (RTR0MemObjGetPagePhysAddr(pMem, iPage++) != NIL_RTHCPHYS) {} */ if (iPage == cPages) return NIL_RTHCPHYS; AssertReturn(iPage < (pMem->cb >> PAGE_SHIFT), NIL_RTHCPHYS); } /* * We know the address of physically contiguous allocations and mappings. */ if (pMem->enmType == RTR0MEMOBJTYPE_CONT) return pMem->u.Cont.Phys + iPage * PAGE_SIZE; if (pMem->enmType == RTR0MEMOBJTYPE_PHYS) return pMem->u.Phys.PhysBase + iPage * PAGE_SIZE; /* * Do the job. */ return rtR0MemObjNativeGetPagePhysAddr(pMem, iPage); } RT_EXPORT_SYMBOL(RTR0MemObjGetPagePhysAddr); /** * Checks whether the allocation was zero initialized or not. * * This only works on allocations. It is not meaningful for mappings, reserved * memory and entered physical address, and will return false for these. * * @returns true if the allocation was initialized to zero at allocation time, * false if not or query not meaningful to the object type. * @param hMemObj The ring-0 memory object to be freed. * * @remarks It can be expected that memory allocated in the same fashion will * have the same initialization state. So, if this returns true for * one allocation it will return true for all other similarly made * allocations. */ RTR0DECL(bool) RTR0MemObjWasZeroInitialized(RTR0MEMOBJ hMemObj) { PRTR0MEMOBJINTERNAL pMem; /* Validate the object handle. */ if (RT_UNLIKELY(hMemObj == NIL_RTR0MEMOBJ)) return false; AssertPtrReturn(hMemObj, false); pMem = (PRTR0MEMOBJINTERNAL)hMemObj; AssertMsgReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, ("%p: %#x\n", pMem, pMem->u32Magic), false); AssertMsgReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, ("%p: %d\n", pMem, pMem->enmType), false); Assert( (pMem->fFlags & (RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC | RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC)) != (RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC | RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC)); /* return the alloc init state. */ return (pMem->fFlags & (RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC | RTR0MEMOBJ_FLAGS_UNINITIALIZED_AT_ALLOC)) == RTR0MEMOBJ_FLAGS_ZERO_AT_ALLOC; } RT_EXPORT_SYMBOL(RTR0MemObjWasZeroInitialized); /** * Frees a ring-0 memory object. * * @returns IPRT status code. * @retval VERR_INVALID_HANDLE if * @param MemObj The ring-0 memory object to be freed. NIL is * accepted. * @param fFreeMappings Whether or not to free mappings of the object. */ RTR0DECL(int) RTR0MemObjFree(RTR0MEMOBJ MemObj, bool fFreeMappings) { /* * Validate the object handle. */ PRTR0MEMOBJINTERNAL pMem; int rc; if (MemObj == NIL_RTR0MEMOBJ) return VINF_SUCCESS; AssertPtrReturn(MemObj, VERR_INVALID_HANDLE); pMem = (PRTR0MEMOBJINTERNAL)MemObj; AssertReturn(pMem->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE); AssertReturn(pMem->enmType > RTR0MEMOBJTYPE_INVALID && pMem->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE); RT_ASSERT_PREEMPTIBLE(); /* * Deal with mappings according to fFreeMappings. */ if ( !rtR0MemObjIsMapping(pMem) && pMem->uRel.Parent.cMappings > 0) { /* fail if not requested to free mappings. */ if (!fFreeMappings) return VERR_MEMORY_BUSY; while (pMem->uRel.Parent.cMappings > 0) { PRTR0MEMOBJINTERNAL pChild = pMem->uRel.Parent.papMappings[--pMem->uRel.Parent.cMappings]; pMem->uRel.Parent.papMappings[pMem->uRel.Parent.cMappings] = NULL; /* sanity checks. */ AssertPtr(pChild); AssertFatal(pChild->u32Magic == RTR0MEMOBJ_MAGIC); AssertFatal(pChild->enmType > RTR0MEMOBJTYPE_INVALID && pChild->enmType < RTR0MEMOBJTYPE_END); AssertFatal(rtR0MemObjIsMapping(pChild)); /* free the mapping. */ rc = rtR0MemObjNativeFree(pChild); if (RT_FAILURE(rc)) { Log(("RTR0MemObjFree: failed to free mapping %p: %p %#zx; rc=%Rrc\n", pChild, pChild->pv, pChild->cb, rc)); pMem->uRel.Parent.papMappings[pMem->uRel.Parent.cMappings++] = pChild; return rc; } pChild->u32Magic++; pChild->enmType = RTR0MEMOBJTYPE_END; RTMemFree(pChild); } } /* * Free this object. */ rc = rtR0MemObjNativeFree(pMem); if (RT_SUCCESS(rc)) { /* * Ok, it was freed just fine. Now, if it's a mapping we'll have to remove it from the parent. */ if (rtR0MemObjIsMapping(pMem)) { PRTR0MEMOBJINTERNAL pParent = pMem->uRel.Child.pParent; uint32_t i; /* sanity checks */ AssertPtr(pParent); AssertFatal(pParent->u32Magic == RTR0MEMOBJ_MAGIC); AssertFatal(pParent->enmType > RTR0MEMOBJTYPE_INVALID && pParent->enmType < RTR0MEMOBJTYPE_END); AssertFatal(!rtR0MemObjIsMapping(pParent)); AssertFatal(pParent->uRel.Parent.cMappings > 0); AssertPtr(pParent->uRel.Parent.papMappings); /* locate and remove from the array of mappings. */ i = pParent->uRel.Parent.cMappings; while (i-- > 0) { if (pParent->uRel.Parent.papMappings[i] == pMem) { pParent->uRel.Parent.papMappings[i] = pParent->uRel.Parent.papMappings[--pParent->uRel.Parent.cMappings]; break; } } Assert(i != UINT32_MAX); } else Assert(pMem->uRel.Parent.cMappings == 0); /* * Finally, destroy the handle. */ pMem->u32Magic++; pMem->enmType = RTR0MEMOBJTYPE_END; if (!rtR0MemObjIsMapping(pMem)) RTMemFree(pMem->uRel.Parent.papMappings); RTMemFree(pMem); } else Log(("RTR0MemObjFree: failed to free %p: %d %p %#zx; rc=%Rrc\n", pMem, pMem->enmType, pMem->pv, pMem->cb, rc)); return rc; } RT_EXPORT_SYMBOL(RTR0MemObjFree); RTR0DECL(int) RTR0MemObjAllocPageTag(PRTR0MEMOBJ pMemObj, size_t cb, bool fExecutable, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeAllocPage(pMemObj, cbAligned, fExecutable, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjAllocPageTag); RTR0DECL(int) RTR0MemObjAllocLargeTag(PRTR0MEMOBJ pMemObj, size_t cb, size_t cbLargePage, uint32_t fFlags, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, cbLargePage); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; #ifdef RT_ARCH_AMD64 AssertReturn(cbLargePage == _2M || cbLargePage == _1G, VERR_OUT_OF_RANGE); #elif defined(RT_ARCH_X86) AssertReturn(cbLargePage == _2M || cbLargePage == _4M, VERR_OUT_OF_RANGE); #else AssertReturn(RT_IS_POWER_OF_TWO(cbLargePage), VERR_NOT_POWER_OF_TWO); AssertReturn(cbLargePage > PAGE_SIZE, VERR_OUT_OF_RANGE); #endif AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); AssertReturn(!(fFlags & ~RTMEMOBJ_ALLOC_LARGE_F_VALID_MASK), VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeAllocLarge(pMemObj, cbAligned, cbLargePage, fFlags, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjAllocLargeTag); /** * Fallback implementation of rtR0MemObjNativeAllocLarge and implements single * page allocation using rtR0MemObjNativeAllocPhys. */ DECLHIDDEN(int) rtR0MemObjFallbackAllocLarge(PPRTR0MEMOBJINTERNAL ppMem, size_t cb, size_t cbLargePage, uint32_t fFlags, const char *pszTag) { RT_NOREF(pszTag, fFlags); if (cb == cbLargePage) return rtR0MemObjNativeAllocPhys(ppMem, cb, NIL_RTHCPHYS, cbLargePage, pszTag); return VERR_NOT_SUPPORTED; } RTR0DECL(int) RTR0MemObjAllocLowTag(PRTR0MEMOBJ pMemObj, size_t cb, bool fExecutable, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeAllocLow(pMemObj, cbAligned, fExecutable, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjAllocLowTag); RTR0DECL(int) RTR0MemObjAllocContTag(PRTR0MEMOBJ pMemObj, size_t cb, bool fExecutable, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeAllocCont(pMemObj, cbAligned, fExecutable, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjAllocContTag); RTR0DECL(int) RTR0MemObjLockUserTag(PRTR0MEMOBJ pMemObj, RTR3PTR R3Ptr, size_t cb, uint32_t fAccess, RTR0PROCESS R0Process, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb + (R3Ptr & PAGE_OFFSET_MASK), PAGE_SIZE); RTR3PTR const R3PtrAligned = (R3Ptr & ~(RTR3PTR)PAGE_OFFSET_MASK); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); if (R0Process == NIL_RTR0PROCESS) R0Process = RTR0ProcHandleSelf(); AssertReturn(!(fAccess & ~(RTMEM_PROT_READ | RTMEM_PROT_WRITE)), VERR_INVALID_PARAMETER); AssertReturn(fAccess, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the locking. */ return rtR0MemObjNativeLockUser(pMemObj, R3PtrAligned, cbAligned, fAccess, R0Process, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjLockUserTag); RTR0DECL(int) RTR0MemObjLockKernelTag(PRTR0MEMOBJ pMemObj, void *pv, size_t cb, uint32_t fAccess, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb + ((uintptr_t)pv & PAGE_OFFSET_MASK), PAGE_SIZE); void * const pvAligned = (void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); AssertPtrReturn(pvAligned, VERR_INVALID_POINTER); AssertReturn(!(fAccess & ~(RTMEM_PROT_READ | RTMEM_PROT_WRITE)), VERR_INVALID_PARAMETER); AssertReturn(fAccess, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeLockKernel(pMemObj, pvAligned, cbAligned, fAccess, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjLockKernelTag); RTR0DECL(int) RTR0MemObjAllocPhysTag(PRTR0MEMOBJ pMemObj, size_t cb, RTHCPHYS PhysHighest, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); AssertReturn(PhysHighest >= cb, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeAllocPhys(pMemObj, cbAligned, PhysHighest, PAGE_SIZE /* page aligned */, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjAllocPhysTag); RTR0DECL(int) RTR0MemObjAllocPhysExTag(PRTR0MEMOBJ pMemObj, size_t cb, RTHCPHYS PhysHighest, size_t uAlignment, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); AssertReturn(PhysHighest >= cb, VERR_INVALID_PARAMETER); if (uAlignment == 0) uAlignment = PAGE_SIZE; AssertReturn( uAlignment == PAGE_SIZE || uAlignment == _2M || uAlignment == _4M || uAlignment == _1G, VERR_INVALID_PARAMETER); #if HC_ARCH_BITS == 32 /* Memory allocated in this way is typically mapped into kernel space as well; simply don't allow this on 32 bits hosts as the kernel space is too crowded already. */ if (uAlignment != PAGE_SIZE) return VERR_NOT_SUPPORTED; #endif RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeAllocPhys(pMemObj, cbAligned, PhysHighest, uAlignment, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjAllocPhysExTag); RTR0DECL(int) RTR0MemObjAllocPhysNCTag(PRTR0MEMOBJ pMemObj, size_t cb, RTHCPHYS PhysHighest, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); AssertReturn(PhysHighest >= cb, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeAllocPhysNC(pMemObj, cbAligned, PhysHighest, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjAllocPhysNCTag); RTR0DECL(int) RTR0MemObjEnterPhysTag(PRTR0MEMOBJ pMemObj, RTHCPHYS Phys, size_t cb, uint32_t uCachePolicy, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb + (Phys & PAGE_OFFSET_MASK), PAGE_SIZE); const RTHCPHYS PhysAligned = Phys & ~(RTHCPHYS)PAGE_OFFSET_MASK; AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); AssertReturn(Phys != NIL_RTHCPHYS, VERR_INVALID_PARAMETER); AssertReturn( uCachePolicy == RTMEM_CACHE_POLICY_DONT_CARE || uCachePolicy == RTMEM_CACHE_POLICY_MMIO, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the allocation. */ return rtR0MemObjNativeEnterPhys(pMemObj, PhysAligned, cbAligned, uCachePolicy, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjEnterPhysTag); RTR0DECL(int) RTR0MemObjReserveKernelTag(PRTR0MEMOBJ pMemObj, void *pvFixed, size_t cb, size_t uAlignment, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; if (uAlignment == 0) uAlignment = PAGE_SIZE; AssertReturn(uAlignment == PAGE_SIZE || uAlignment == _2M || uAlignment == _4M, VERR_INVALID_PARAMETER); AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); if (pvFixed != (void *)-1) AssertReturn(!((uintptr_t)pvFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the reservation. */ return rtR0MemObjNativeReserveKernel(pMemObj, pvFixed, cbAligned, uAlignment, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjReserveKernelTag); RTR0DECL(int) RTR0MemObjReserveUserTag(PRTR0MEMOBJ pMemObj, RTR3PTR R3PtrFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process, const char *pszTag) { /* sanity checks. */ const size_t cbAligned = RT_ALIGN_Z(cb, PAGE_SIZE); AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; if (uAlignment == 0) uAlignment = PAGE_SIZE; AssertReturn(uAlignment == PAGE_SIZE || uAlignment == _2M || uAlignment == _4M, VERR_INVALID_PARAMETER); AssertReturn(cb > 0, VERR_INVALID_PARAMETER); AssertReturn(cb <= cbAligned, VERR_INVALID_PARAMETER); if (R3PtrFixed != (RTR3PTR)-1) AssertReturn(!(R3PtrFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER); if (R0Process == NIL_RTR0PROCESS) R0Process = RTR0ProcHandleSelf(); RT_ASSERT_PREEMPTIBLE(); /* do the reservation. */ return rtR0MemObjNativeReserveUser(pMemObj, R3PtrFixed, cbAligned, uAlignment, R0Process, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjReserveUserTag); RTR0DECL(int) RTR0MemObjMapKernelTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, void *pvFixed, size_t uAlignment, unsigned fProt, const char *pszTag) { return RTR0MemObjMapKernelExTag(pMemObj, MemObjToMap, pvFixed, uAlignment, fProt, 0, 0, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjMapKernelTag); RTR0DECL(int) RTR0MemObjMapKernelExTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, void *pvFixed, size_t uAlignment, unsigned fProt, size_t offSub, size_t cbSub, const char *pszTag) { PRTR0MEMOBJINTERNAL pMemToMap; PRTR0MEMOBJINTERNAL pNew; int rc; /* sanity checks. */ AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); *pMemObj = NIL_RTR0MEMOBJ; AssertPtrReturn(MemObjToMap, VERR_INVALID_HANDLE); pMemToMap = (PRTR0MEMOBJINTERNAL)MemObjToMap; AssertReturn(pMemToMap->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE); AssertReturn(pMemToMap->enmType > RTR0MEMOBJTYPE_INVALID && pMemToMap->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE); AssertReturn(!rtR0MemObjIsMapping(pMemToMap), VERR_INVALID_PARAMETER); AssertReturn(pMemToMap->enmType != RTR0MEMOBJTYPE_RES_VIRT, VERR_INVALID_PARAMETER); if (uAlignment == 0) uAlignment = PAGE_SIZE; AssertReturn(uAlignment == PAGE_SIZE || uAlignment == _2M || uAlignment == _4M, VERR_INVALID_PARAMETER); if (pvFixed != (void *)-1) AssertReturn(!((uintptr_t)pvFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER); AssertReturn(fProt != RTMEM_PROT_NONE, VERR_INVALID_PARAMETER); AssertReturn(!(fProt & ~(RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC)), VERR_INVALID_PARAMETER); AssertReturn(!(offSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER); AssertReturn(offSub < pMemToMap->cb, VERR_INVALID_PARAMETER); AssertReturn(!(cbSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER); AssertReturn(cbSub <= pMemToMap->cb, VERR_INVALID_PARAMETER); AssertReturn((!offSub && !cbSub) || (offSub + cbSub) <= pMemToMap->cb, VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* adjust the request to simplify the native code. */ if (offSub == 0 && cbSub == pMemToMap->cb) cbSub = 0; /* do the mapping. */ rc = rtR0MemObjNativeMapKernel(&pNew, pMemToMap, pvFixed, uAlignment, fProt, offSub, cbSub, pszTag); if (RT_SUCCESS(rc)) { /* link it. */ rc = rtR0MemObjLink(pMemToMap, pNew); if (RT_SUCCESS(rc)) *pMemObj = pNew; else { /* damn, out of memory. bail out. */ int rc2 = rtR0MemObjNativeFree(pNew); AssertRC(rc2); pNew->u32Magic++; pNew->enmType = RTR0MEMOBJTYPE_END; RTMemFree(pNew); } } return rc; } RT_EXPORT_SYMBOL(RTR0MemObjMapKernelExTag); RTR0DECL(int) RTR0MemObjMapUserTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, RTR3PTR R3PtrFixed, size_t uAlignment, unsigned fProt, RTR0PROCESS R0Process, const char *pszTag) { return RTR0MemObjMapUserExTag(pMemObj, MemObjToMap, R3PtrFixed, uAlignment, fProt, R0Process, 0, 0, pszTag); } RT_EXPORT_SYMBOL(RTR0MemObjMapUserTag); RTR0DECL(int) RTR0MemObjMapUserExTag(PRTR0MEMOBJ pMemObj, RTR0MEMOBJ MemObjToMap, RTR3PTR R3PtrFixed, size_t uAlignment, unsigned fProt, RTR0PROCESS R0Process, size_t offSub, size_t cbSub, const char *pszTag) { /* sanity checks. */ PRTR0MEMOBJINTERNAL pMemToMap; PRTR0MEMOBJINTERNAL pNew; int rc; AssertPtrReturn(pMemObj, VERR_INVALID_POINTER); pMemToMap = (PRTR0MEMOBJINTERNAL)MemObjToMap; *pMemObj = NIL_RTR0MEMOBJ; AssertPtrReturn(MemObjToMap, VERR_INVALID_HANDLE); AssertReturn(pMemToMap->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE); AssertReturn(pMemToMap->enmType > RTR0MEMOBJTYPE_INVALID && pMemToMap->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE); AssertReturn(!rtR0MemObjIsMapping(pMemToMap), VERR_INVALID_PARAMETER); AssertReturn(pMemToMap->enmType != RTR0MEMOBJTYPE_RES_VIRT, VERR_INVALID_PARAMETER); if (uAlignment == 0) uAlignment = PAGE_SIZE; AssertReturn(uAlignment == PAGE_SIZE || uAlignment == _2M || uAlignment == _4M, VERR_INVALID_PARAMETER); if (R3PtrFixed != (RTR3PTR)-1) AssertReturn(!(R3PtrFixed & (uAlignment - 1)), VERR_INVALID_PARAMETER); AssertReturn(fProt != RTMEM_PROT_NONE, VERR_INVALID_PARAMETER); AssertReturn(!(fProt & ~(RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC)), VERR_INVALID_PARAMETER); AssertReturn(!(offSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER); AssertReturn(offSub < pMemToMap->cb, VERR_INVALID_PARAMETER); AssertReturn(!(cbSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER); AssertReturn(cbSub <= pMemToMap->cb, VERR_INVALID_PARAMETER); AssertReturn((!offSub && !cbSub) || (offSub + cbSub) <= pMemToMap->cb, VERR_INVALID_PARAMETER); if (R0Process == NIL_RTR0PROCESS) R0Process = RTR0ProcHandleSelf(); RT_ASSERT_PREEMPTIBLE(); /* adjust the request to simplify the native code. */ if (offSub == 0 && cbSub == pMemToMap->cb) cbSub = 0; /* do the mapping. */ rc = rtR0MemObjNativeMapUser(&pNew, pMemToMap, R3PtrFixed, uAlignment, fProt, R0Process, offSub, cbSub, pszTag); if (RT_SUCCESS(rc)) { /* link it. */ rc = rtR0MemObjLink(pMemToMap, pNew); if (RT_SUCCESS(rc)) *pMemObj = pNew; else { /* damn, out of memory. bail out. */ int rc2 = rtR0MemObjNativeFree(pNew); AssertRC(rc2); pNew->u32Magic++; pNew->enmType = RTR0MEMOBJTYPE_END; RTMemFree(pNew); } } return rc; } RT_EXPORT_SYMBOL(RTR0MemObjMapUserExTag); RTR0DECL(int) RTR0MemObjProtect(RTR0MEMOBJ hMemObj, size_t offSub, size_t cbSub, uint32_t fProt) { PRTR0MEMOBJINTERNAL pMemObj; int rc; /* sanity checks. */ pMemObj = (PRTR0MEMOBJINTERNAL)hMemObj; AssertPtrReturn(pMemObj, VERR_INVALID_HANDLE); AssertReturn(pMemObj->u32Magic == RTR0MEMOBJ_MAGIC, VERR_INVALID_HANDLE); AssertReturn(pMemObj->enmType > RTR0MEMOBJTYPE_INVALID && pMemObj->enmType < RTR0MEMOBJTYPE_END, VERR_INVALID_HANDLE); AssertReturn(rtR0MemObjIsProtectable(pMemObj), VERR_INVALID_PARAMETER); AssertReturn(!(offSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER); AssertReturn(offSub < pMemObj->cb, VERR_INVALID_PARAMETER); AssertReturn(!(cbSub & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER); AssertReturn(cbSub <= pMemObj->cb, VERR_INVALID_PARAMETER); AssertReturn(offSub + cbSub <= pMemObj->cb, VERR_INVALID_PARAMETER); AssertReturn(!(fProt & ~(RTMEM_PROT_NONE | RTMEM_PROT_READ | RTMEM_PROT_WRITE | RTMEM_PROT_EXEC)), VERR_INVALID_PARAMETER); RT_ASSERT_PREEMPTIBLE(); /* do the job */ rc = rtR0MemObjNativeProtect(pMemObj, offSub, cbSub, fProt); if (RT_SUCCESS(rc)) pMemObj->fFlags |= RTR0MEMOBJ_FLAGS_PROT_CHANGED; /* record it */ return rc; } RT_EXPORT_SYMBOL(RTR0MemObjProtect);