/* $Id: critsect-generic.cpp $ */ /** @file * IPRT - Critical Section, Generic. */ /* * Copyright (C) 2006-2023 Oracle and/or its affiliates. * * This file is part of VirtualBox base platform packages, as * available from https://www.virtualbox.org. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation, in version 3 of the * License. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, see . * * The contents of this file may alternatively be used under the terms * of the Common Development and Distribution License Version 1.0 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included * in the VirtualBox distribution, in which case the provisions of the * CDDL are applicable instead of those of the GPL. * * You may elect to license modified versions of this file under the * terms and conditions of either the GPL or the CDDL or both. * * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 */ /********************************************************************************************************************************* * Header Files * *********************************************************************************************************************************/ #define RTCRITSECT_WITHOUT_REMAPPING #include #include "internal/iprt.h" #include #include #include #include #include #include "internal/thread.h" #include "internal/strict.h" /* Two issues here, (1) the tracepoint generator uses IPRT, and (2) only one .d file per module. */ #ifdef IPRT_WITH_DTRACE # include IPRT_DTRACE_INCLUDE # ifdef IPRT_DTRACE_PREFIX # define IPRT_CRITSECT_ENTERED RT_CONCAT(IPRT_DTRACE_PREFIX,IPRT_CRITSECT_ENTERED) # define IPRT_CRITSECT_LEAVING RT_CONCAT(IPRT_DTRACE_PREFIX,IPRT_CRITSECT_LEAVING) # define IPRT_CRITSECT_BUSY RT_CONCAT(IPRT_DTRACE_PREFIX,IPRT_CRITSECT_BUSY) # define IPRT_CRITSECT_WAITING RT_CONCAT(IPRT_DTRACE_PREFIX,IPRT_CRITSECT_WAITING) # endif #else # define IPRT_CRITSECT_ENTERED(a_pvCritSect, a_pszName, a_cLockers, a_cNestings) do {} while (0) # define IPRT_CRITSECT_LEAVING(a_pvCritSect, a_pszName, a_cLockers, a_cNestings) do {} while (0) # define IPRT_CRITSECT_BUSY( a_pvCritSect, a_pszName, a_cLockers, a_pvNativeOwnerThread) do {} while (0) # define IPRT_CRITSECT_WAITING(a_pvCritSect, a_pszName, a_cLockers, a_pvNativeOwnerThread) do {} while (0) #endif RTDECL(int) RTCritSectInit(PRTCRITSECT pCritSect) { return RTCritSectInitEx(pCritSect, 0, NIL_RTLOCKVALCLASS, RTLOCKVAL_SUB_CLASS_NONE, "RTCritSect"); } RT_EXPORT_SYMBOL(RTCritSectInit); RTDECL(int) RTCritSectInitEx(PRTCRITSECT pCritSect, uint32_t fFlags, RTLOCKVALCLASS hClass, uint32_t uSubClass, const char *pszNameFmt, ...) { AssertReturn(!(fFlags & ~(RTCRITSECT_FLAGS_NO_NESTING | RTCRITSECT_FLAGS_NO_LOCK_VAL | RTCRITSECT_FLAGS_BOOTSTRAP_HACK | RTCRITSECT_FLAGS_NOP)), VERR_INVALID_PARAMETER); RT_NOREF_PV(hClass); RT_NOREF_PV(uSubClass); RT_NOREF_PV(pszNameFmt); /* * Initialize the structure and */ pCritSect->u32Magic = RTCRITSECT_MAGIC; #ifdef IN_RING0 pCritSect->fFlags = fFlags | RTCRITSECT_FLAGS_RING0; #else pCritSect->fFlags = fFlags & ~RTCRITSECT_FLAGS_RING0; #endif pCritSect->cNestings = 0; pCritSect->cLockers = -1; pCritSect->NativeThreadOwner = NIL_RTNATIVETHREAD; pCritSect->pValidatorRec = NULL; int rc = VINF_SUCCESS; #ifdef RTCRITSECT_STRICT if (!(fFlags & (RTCRITSECT_FLAGS_BOOTSTRAP_HACK | RTCRITSECT_FLAGS_NOP))) { if (!pszNameFmt) { static uint32_t volatile s_iCritSectAnon = 0; rc = RTLockValidatorRecExclCreate(&pCritSect->pValidatorRec, hClass, uSubClass, pCritSect, !(fFlags & RTCRITSECT_FLAGS_NO_LOCK_VAL), "RTCritSect-%u", ASMAtomicIncU32(&s_iCritSectAnon) - 1); } else { va_list va; va_start(va, pszNameFmt); rc = RTLockValidatorRecExclCreateV(&pCritSect->pValidatorRec, hClass, uSubClass, pCritSect, !(fFlags & RTCRITSECT_FLAGS_NO_LOCK_VAL), pszNameFmt, va); va_end(va); } } #endif if (RT_SUCCESS(rc)) { #ifdef IN_RING0 rc = RTSemEventCreate(&pCritSect->EventSem); #else rc = RTSemEventCreateEx(&pCritSect->EventSem, fFlags & RTCRITSECT_FLAGS_BOOTSTRAP_HACK ? RTSEMEVENT_FLAGS_NO_LOCK_VAL | RTSEMEVENT_FLAGS_BOOTSTRAP_HACK : RTSEMEVENT_FLAGS_NO_LOCK_VAL, NIL_RTLOCKVALCLASS, NULL); #endif if (RT_SUCCESS(rc)) return VINF_SUCCESS; #ifdef RTCRITSECT_STRICT RTLockValidatorRecExclDestroy(&pCritSect->pValidatorRec); #endif } AssertRC(rc); pCritSect->EventSem = NULL; pCritSect->u32Magic = (uint32_t)rc; return rc; } RT_EXPORT_SYMBOL(RTCritSectInitEx); RTDECL(uint32_t) RTCritSectSetSubClass(PRTCRITSECT pCritSect, uint32_t uSubClass) { # ifdef RTCRITSECT_STRICT AssertPtrReturn(pCritSect, RTLOCKVAL_SUB_CLASS_INVALID); AssertReturn(pCritSect->u32Magic == RTCRITSECT_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID); AssertReturn(!(pCritSect->fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID); return RTLockValidatorRecExclSetSubClass(pCritSect->pValidatorRec, uSubClass); # else RT_NOREF_PV(pCritSect); RT_NOREF_PV(uSubClass); return RTLOCKVAL_SUB_CLASS_INVALID; # endif } DECL_FORCE_INLINE(int) rtCritSectTryEnter(PRTCRITSECT pCritSect, PCRTLOCKVALSRCPOS pSrcPos) { Assert(pCritSect); Assert(pCritSect->u32Magic == RTCRITSECT_MAGIC); /*AssertReturn(pCritSect->u32Magic == RTCRITSECT_MAGIC, VERR_SEM_DESTROYED);*/ #ifdef IN_RING0 Assert(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0); #else Assert(!(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0)); #endif RT_NOREF_PV(pSrcPos); /* * Return straight away if NOP. */ if (pCritSect->fFlags & RTCRITSECT_FLAGS_NOP) return VINF_SUCCESS; /* * Try take the lock. (cLockers is -1 if it's free) */ RTNATIVETHREAD NativeThreadSelf = RTThreadNativeSelf(); if (!ASMAtomicCmpXchgS32(&pCritSect->cLockers, 0, -1)) { /* * Somebody is owning it (or will be soon). Perhaps it's us? */ if (pCritSect->NativeThreadOwner == NativeThreadSelf) { if (!(pCritSect->fFlags & RTCRITSECT_FLAGS_NO_NESTING)) { #ifdef RTCRITSECT_STRICT int rc9 = RTLockValidatorRecExclRecursion(pCritSect->pValidatorRec, pSrcPos); if (RT_FAILURE(rc9)) return rc9; #endif int32_t cLockers = ASMAtomicIncS32(&pCritSect->cLockers); NOREF(cLockers); pCritSect->cNestings++; IPRT_CRITSECT_ENTERED(pCritSect, NULL, cLockers, pCritSect->cNestings); return VINF_SUCCESS; } AssertMsgFailed(("Nested entry of critsect %p\n", pCritSect)); return VERR_SEM_NESTED; } IPRT_CRITSECT_BUSY(pCritSect, NULL, pCritSect->cLockers, (void *)pCritSect->NativeThreadOwner); return VERR_SEM_BUSY; } /* * First time */ pCritSect->cNestings = 1; ASMAtomicWriteHandle(&pCritSect->NativeThreadOwner, NativeThreadSelf); #ifdef RTCRITSECT_STRICT RTLockValidatorRecExclSetOwner(pCritSect->pValidatorRec, NIL_RTTHREAD, pSrcPos, true); #endif IPRT_CRITSECT_ENTERED(pCritSect, NULL, 0, 1); return VINF_SUCCESS; } RTDECL(int) RTCritSectTryEnter(PRTCRITSECT pCritSect) { #ifndef RTCRTISECT_STRICT return rtCritSectTryEnter(pCritSect, NULL); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return rtCritSectTryEnter(pCritSect, &SrcPos); #endif } RT_EXPORT_SYMBOL(RTCritSectTryEnter); RTDECL(int) RTCritSectTryEnterDebug(PRTCRITSECT pCritSect, RTHCUINTPTR uId, RT_SRC_POS_DECL) { RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return rtCritSectTryEnter(pCritSect, &SrcPos); } RT_EXPORT_SYMBOL(RTCritSectTryEnterDebug); DECL_FORCE_INLINE(int) rtCritSectEnter(PRTCRITSECT pCritSect, PCRTLOCKVALSRCPOS pSrcPos) { AssertPtr(pCritSect); AssertReturn(pCritSect->u32Magic == RTCRITSECT_MAGIC, VERR_SEM_DESTROYED); #ifdef IN_RING0 Assert(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0); #else Assert(!(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0)); #endif RT_NOREF_PV(pSrcPos); /* * Return straight away if NOP. */ if (pCritSect->fFlags & RTCRITSECT_FLAGS_NOP) return VINF_SUCCESS; /* * How is calling and is the order right? */ RTNATIVETHREAD NativeThreadSelf = RTThreadNativeSelf(); #ifdef RTCRITSECT_STRICT RTTHREAD hThreadSelf = pCritSect->pValidatorRec ? RTThreadSelfAutoAdopt() : RTThreadSelf(); int rc9; if (pCritSect->pValidatorRec) /* (bootstap) */ { rc9 = RTLockValidatorRecExclCheckOrder(pCritSect->pValidatorRec, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); if (RT_FAILURE(rc9)) return rc9; } #endif /* * Increment the waiter counter. * This becomes 0 when the section is free. */ int32_t cLockers = ASMAtomicIncS32(&pCritSect->cLockers); if (cLockers > 0) { /* * Nested? */ if (pCritSect->NativeThreadOwner == NativeThreadSelf) { if (!(pCritSect->fFlags & RTCRITSECT_FLAGS_NO_NESTING)) { #ifdef RTCRITSECT_STRICT rc9 = RTLockValidatorRecExclRecursion(pCritSect->pValidatorRec, pSrcPos); if (RT_FAILURE(rc9)) { ASMAtomicDecS32(&pCritSect->cLockers); return rc9; } #endif pCritSect->cNestings++; IPRT_CRITSECT_ENTERED(pCritSect, NULL, cLockers, pCritSect->cNestings); return VINF_SUCCESS; } AssertBreakpoint(); /* don't do normal assertion here, the logger uses this code too. */ ASMAtomicDecS32(&pCritSect->cLockers); return VERR_SEM_NESTED; } /* * Wait for the current owner to release it. */ IPRT_CRITSECT_WAITING(pCritSect, NULL, cLockers, (void *)pCritSect->NativeThreadOwner); #if !defined(RTCRITSECT_STRICT) && defined(IN_RING3) RTTHREAD hThreadSelf = RTThreadSelf(); #endif for (;;) { #ifdef RTCRITSECT_STRICT rc9 = RTLockValidatorRecExclCheckBlocking(pCritSect->pValidatorRec, hThreadSelf, pSrcPos, !(pCritSect->fFlags & RTCRITSECT_FLAGS_NO_NESTING), RT_INDEFINITE_WAIT, RTTHREADSTATE_CRITSECT, false); if (RT_FAILURE(rc9)) { ASMAtomicDecS32(&pCritSect->cLockers); return rc9; } #elif defined(IN_RING3) RTThreadBlocking(hThreadSelf, RTTHREADSTATE_CRITSECT, false); #endif int rc = RTSemEventWait(pCritSect->EventSem, RT_INDEFINITE_WAIT); #ifdef IN_RING3 RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_CRITSECT); #endif if (pCritSect->u32Magic != RTCRITSECT_MAGIC) return VERR_SEM_DESTROYED; if (rc == VINF_SUCCESS) break; AssertMsg(rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED, ("rc=%Rrc\n", rc)); } AssertMsg(pCritSect->NativeThreadOwner == NIL_RTNATIVETHREAD, ("pCritSect->NativeThreadOwner=%p\n", pCritSect->NativeThreadOwner)); } /* * First time */ pCritSect->cNestings = 1; ASMAtomicWriteHandle(&pCritSect->NativeThreadOwner, NativeThreadSelf); #ifdef RTCRITSECT_STRICT RTLockValidatorRecExclSetOwner(pCritSect->pValidatorRec, hThreadSelf, pSrcPos, true); #endif IPRT_CRITSECT_ENTERED(pCritSect, NULL, 0, 1); return VINF_SUCCESS; } RTDECL(int) RTCritSectEnter(PRTCRITSECT pCritSect) { #ifndef RTCRITSECT_STRICT return rtCritSectEnter(pCritSect, NULL); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return rtCritSectEnter(pCritSect, &SrcPos); #endif } RT_EXPORT_SYMBOL(RTCritSectEnter); RTDECL(int) RTCritSectEnterDebug(PRTCRITSECT pCritSect, RTHCUINTPTR uId, RT_SRC_POS_DECL) { RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return rtCritSectEnter(pCritSect, &SrcPos); } RT_EXPORT_SYMBOL(RTCritSectEnterDebug); RTDECL(int) RTCritSectLeave(PRTCRITSECT pCritSect) { /* * Assert sanity and check for NOP. */ Assert(pCritSect); Assert(pCritSect->u32Magic == RTCRITSECT_MAGIC); #ifdef IN_RING0 Assert(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0); #else Assert(!(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0)); #endif if (pCritSect->fFlags & RTCRITSECT_FLAGS_NOP) return VINF_SUCCESS; /* * Assert ownership and so on. */ Assert(pCritSect->cNestings > 0); Assert(pCritSect->cLockers >= 0); Assert(pCritSect->NativeThreadOwner == RTThreadNativeSelf()); #ifdef RTCRITSECT_STRICT int rc9 = RTLockValidatorRecExclReleaseOwner(pCritSect->pValidatorRec, pCritSect->cNestings == 1); if (RT_FAILURE(rc9)) return rc9; #endif /* * Decrement nestings, if <= 0 when we'll release the critsec. */ uint32_t cNestings = --pCritSect->cNestings; IPRT_CRITSECT_LEAVING(pCritSect, NULL, ASMAtomicUoReadS32(&pCritSect->cLockers) - 1, cNestings); if (cNestings > 0) ASMAtomicDecS32(&pCritSect->cLockers); else { /* * Set owner to zero. * Decrement waiters, if >= 0 then we have to wake one of them up. */ ASMAtomicWriteHandle(&pCritSect->NativeThreadOwner, NIL_RTNATIVETHREAD); if (ASMAtomicDecS32(&pCritSect->cLockers) >= 0) { int rc = RTSemEventSignal(pCritSect->EventSem); AssertReleaseMsgRC(rc, ("RTSemEventSignal -> %Rrc\n", rc)); } } return VINF_SUCCESS; } RT_EXPORT_SYMBOL(RTCritSectLeave); #ifdef IN_RING3 static int rtCritSectEnterMultiple(size_t cCritSects, PRTCRITSECT *papCritSects, PCRTLOCKVALSRCPOS pSrcPos) { Assert(cCritSects > 0); AssertPtr(papCritSects); /* * Try get them all. */ int rc = VERR_INVALID_PARAMETER; size_t i; for (i = 0; i < cCritSects; i++) { rc = rtCritSectTryEnter(papCritSects[i], pSrcPos); if (RT_FAILURE(rc)) break; } if (RT_SUCCESS(rc)) return rc; /* * The retry loop. */ for (unsigned cTries = 0; ; cTries++) { /* * We've failed, release any locks we might have gotten. ('i' is the lock that failed btw.) */ size_t j = i; while (j-- > 0) { int rc2 = RTCritSectLeave(papCritSects[j]); AssertRC(rc2); } if (rc != VERR_SEM_BUSY) return rc; /* * Try prevent any theoretical synchronous races with other threads. */ Assert(cTries < 1000000); if (cTries > 10000) RTThreadSleep(cTries % 3); /* * Wait on the one we failed to get. */ rc = rtCritSectEnter(papCritSects[i], pSrcPos); if (RT_FAILURE(rc)) return rc; /* * Try take the others. */ for (j = 0; j < cCritSects; j++) { if (j != i) { rc = rtCritSectTryEnter(papCritSects[j], pSrcPos); if (RT_FAILURE(rc)) break; } } if (RT_SUCCESS(rc)) return rc; /* * We failed. */ if (i > j) { int rc2 = RTCritSectLeave(papCritSects[i]); AssertRC(rc2); } i = j; } } RTDECL(int) RTCritSectEnterMultiple(size_t cCritSects, PRTCRITSECT *papCritSects) { #ifndef RTCRITSECT_STRICT return rtCritSectEnterMultiple(cCritSects, papCritSects, NULL); #else RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); return rtCritSectEnterMultiple(cCritSects, papCritSects, &SrcPos); #endif } RT_EXPORT_SYMBOL(RTCritSectEnterMultiple); RTDECL(int) RTCritSectEnterMultipleDebug(size_t cCritSects, PRTCRITSECT *papCritSects, RTHCUINTPTR uId, RT_SRC_POS_DECL) { RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); return rtCritSectEnterMultiple(cCritSects, papCritSects, &SrcPos); } RT_EXPORT_SYMBOL(RTCritSectEnterMultipleDebug); RTDECL(int) RTCritSectLeaveMultiple(size_t cCritSects, PRTCRITSECT *papCritSects) { int rc = VINF_SUCCESS; for (size_t i = 0; i < cCritSects; i++) { int rc2 = RTCritSectLeave(papCritSects[i]); if (RT_FAILURE(rc2) && RT_SUCCESS(rc)) rc = rc2; } return rc; } RT_EXPORT_SYMBOL(RTCritSectLeaveMultiple); #endif /* IN_RING3 */ RTDECL(int) RTCritSectDelete(PRTCRITSECT pCritSect) { /* * Assert free waiters and so on. */ Assert(pCritSect); Assert(pCritSect->u32Magic == RTCRITSECT_MAGIC); Assert(pCritSect->cNestings == 0); Assert(pCritSect->cLockers == -1); Assert(pCritSect->NativeThreadOwner == NIL_RTNATIVETHREAD); #ifdef IN_RING0 Assert(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0); #else Assert(!(pCritSect->fFlags & RTCRITSECT_FLAGS_RING0)); #endif /* * Invalidate the structure and free the mutex. * In case someone is waiting we'll signal the semaphore cLockers + 1 times. */ ASMAtomicWriteU32(&pCritSect->u32Magic, ~RTCRITSECT_MAGIC); pCritSect->fFlags = 0; pCritSect->cNestings = 0; pCritSect->NativeThreadOwner= NIL_RTNATIVETHREAD; RTSEMEVENT EventSem = pCritSect->EventSem; pCritSect->EventSem = NIL_RTSEMEVENT; while (pCritSect->cLockers-- >= 0) RTSemEventSignal(EventSem); ASMAtomicWriteS32(&pCritSect->cLockers, -1); int rc = RTSemEventDestroy(EventSem); AssertRC(rc); #ifdef RTCRITSECT_STRICT RTLockValidatorRecExclDestroy(&pCritSect->pValidatorRec); #endif return rc; } RT_EXPORT_SYMBOL(RTCritSectDelete);