/* $Id: timer-r0drv-linux.c $ */ /** @file * IPRT - Timers, Ring-0 Driver, Linux. */ /* * Copyright (C) 2006-2019 Oracle Corporation * * This file is part of VirtualBox Open Source Edition (OSE), as * available from http://www.virtualbox.org. This file is free software; * you can redistribute it and/or modify it under the terms of the GNU * General Public License (GPL) as published by the Free Software * Foundation, in version 2 as it comes in the "COPYING" file of the * VirtualBox OSE distribution. VirtualBox OSE is distributed in the * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. * * The contents of this file may alternatively be used under the terms * of the Common Development and Distribution License Version 1.0 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the * VirtualBox OSE distribution, in which case the provisions of the * CDDL are applicable instead of those of the GPL. * * You may elect to license modified versions of this file under the * terms and conditions of either the GPL or the CDDL or both. */ /********************************************************************************************************************************* * Header Files * *********************************************************************************************************************************/ #include "the-linux-kernel.h" #include "internal/iprt.h" #include #include #include #include #include #include #include #include #include #include "internal/magics.h" /** @def RTTIMER_LINUX_WITH_HRTIMER * Whether to use high resolution timers. */ #if !defined(RTTIMER_LINUX_WITH_HRTIMER) \ && defined(IPRT_LINUX_HAS_HRTIMER) # define RTTIMER_LINUX_WITH_HRTIMER #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31) # define mod_timer_pinned mod_timer # define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS #endif /********************************************************************************************************************************* * Structures and Typedefs * *********************************************************************************************************************************/ /** * Timer state machine. * * This is used to try handle the issues with MP events and * timers that runs on all CPUs. It's relatively nasty :-/ */ typedef enum RTTIMERLNXSTATE { /** Stopped. */ RTTIMERLNXSTATE_STOPPED = 0, /** Transient state; next ACTIVE. */ RTTIMERLNXSTATE_STARTING, /** Transient state; next ACTIVE. (not really necessary) */ RTTIMERLNXSTATE_MP_STARTING, /** Active. */ RTTIMERLNXSTATE_ACTIVE, /** Active and in callback; next ACTIVE, STOPPED or CALLBACK_DESTROYING. */ RTTIMERLNXSTATE_CALLBACK, /** Stopped while in the callback; next STOPPED. */ RTTIMERLNXSTATE_CB_STOPPING, /** Restarted while in the callback; next ACTIVE, STOPPED, DESTROYING. */ RTTIMERLNXSTATE_CB_RESTARTING, /** The callback shall destroy the timer; next STOPPED. */ RTTIMERLNXSTATE_CB_DESTROYING, /** Transient state; next STOPPED. */ RTTIMERLNXSTATE_STOPPING, /** Transient state; next STOPPED. */ RTTIMERLNXSTATE_MP_STOPPING, /** The usual 32-bit hack. */ RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff } RTTIMERLNXSTATE; /** * A Linux sub-timer. */ typedef struct RTTIMERLNXSUBTIMER { /** Timer specific data. */ union { #if defined(RTTIMER_LINUX_WITH_HRTIMER) /** High resolution timer. */ struct { /** The linux timer structure. */ struct hrtimer LnxTimer; } Hr; #endif /** Standard timer. */ struct { /** The linux timer structure. */ struct timer_list LnxTimer; /** The start of the current run (ns). * This is used to calculate when the timer ought to fire the next time. */ uint64_t u64NextTS; /** The u64NextTS in jiffies. */ unsigned long ulNextJiffies; /** Set when starting or changing the timer so that u64StartTs * and u64NextTS gets reinitialized (eliminating some jitter). */ bool volatile fFirstAfterChg; } Std; } u; /** The current tick number. */ uint64_t iTick; /** Restart the single shot timer at this specific time. * Used when a single shot timer is restarted from the callback. */ uint64_t volatile uNsRestartAt; /** Pointer to the parent timer. */ PRTTIMER pParent; /** The current sub-timer state. */ RTTIMERLNXSTATE volatile enmState; } RTTIMERLNXSUBTIMER; /** Pointer to a linux sub-timer. */ typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER; /** * The internal representation of an Linux timer handle. */ typedef struct RTTIMER { /** Magic. * This is RTTIMER_MAGIC, but changes to something else before the timer * is destroyed to indicate clearly that thread should exit. */ uint32_t volatile u32Magic; /** Spinlock synchronizing the fSuspended and MP event handling. * This is NIL_RTSPINLOCK if cCpus == 1. */ RTSPINLOCK hSpinlock; /** Flag indicating that the timer is suspended. */ bool volatile fSuspended; /** Whether the timer must run on one specific CPU or not. */ bool fSpecificCpu; #ifdef CONFIG_SMP /** Whether the timer must run on all CPUs or not. */ bool fAllCpus; #endif /* else: All -> specific on non-SMP kernels */ /** Whether it is a high resolution timer or a standard one. */ bool fHighRes; /** The id of the CPU it must run on if fSpecificCpu is set. */ RTCPUID idCpu; /** The number of CPUs this timer should run on. */ RTCPUID cCpus; /** Callback. */ PFNRTTIMER pfnTimer; /** User argument. */ void *pvUser; /** The timer interval. 0 if one-shot. */ uint64_t volatile u64NanoInterval; /** This is set to the number of jiffies between ticks if the interval is * an exact number of jiffies. (Standard timers only.) */ unsigned long volatile cJiffies; /** The change interval spinlock for standard timers only. */ spinlock_t ChgIntLock; /** Workqueue item for delayed destruction. */ RTR0LNXWORKQUEUEITEM DtorWorkqueueItem; /** Sub-timers. * Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain * an entry for all possible cpus. In that case the index will be the same as * for the RTCpuSet. */ RTTIMERLNXSUBTIMER aSubTimers[1]; } RTTIMER; /** * A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package. */ typedef struct RTTIMERLINUXSTARTONCPUARGS { /** The current time (RTTimeSystemNanoTS). */ uint64_t u64Now; /** When to start firing (delta). */ uint64_t u64First; } RTTIMERLINUXSTARTONCPUARGS; /** Pointer to a rtTimerLinuxStartOnCpu argument package. */ typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS; /********************************************************************************************************************************* * Internal Functions * *********************************************************************************************************************************/ #ifdef CONFIG_SMP static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser); #endif #if 0 #define DEBUG_HACKING #include #include static void myLogBackdoorPrintf(const char *pszFormat, ...) { char szTmp[256]; va_list args; size_t cb; cb = RTStrPrintf(szTmp, sizeof(szTmp) - 10, "%d: ", RTMpCpuId()); va_start(args, pszFormat); cb += RTStrPrintfV(&szTmp[cb], sizeof(szTmp) - cb, pszFormat, args); va_end(args); ASMOutStrU8(0x504, (uint8_t *)&szTmp[0], cb); } # define RTAssertMsg1Weak(pszExpr, uLine, pszFile, pszFunction) \ myLogBackdoorPrintf("\n!!Guest Assertion failed!!\n%s(%d) %s\n%s\n", uLine, pszFile, pszFunction, (pszExpr)) # define RTAssertMsg2Weak myLogBackdoorPrintf # define RTTIMERLNX_LOG(a) myLogBackdoorPrintf a #else # define RTTIMERLNX_LOG(a) do { } while (0) #endif /** * Sets the state. */ DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState) { #ifdef DEBUG_HACKING RTTIMERLNX_LOG(("set %d -> %d\n", *penmState, enmNewState)); #endif ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState); } /** * Sets the state if it has a certain value. * * @return true if xchg was done. * @return false if xchg wasn't done. */ #ifdef DEBUG_HACKING #define rtTimerLnxCmpXchgState(penmState, enmNewState, enmCurState) rtTimerLnxCmpXchgStateDebug(penmState, enmNewState, enmCurState, __LINE__) static bool rtTimerLnxCmpXchgStateDebug(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState, uint32_t uLine) { RTTIMERLNXSTATE enmOldState = enmCurState; bool fRc = ASMAtomicCmpXchgExU32((uint32_t volatile *)penmState, enmNewState, enmCurState, (uint32_t *)&enmOldState); RTTIMERLNX_LOG(("cxg %d -> %d - %d at %u\n", enmOldState, enmNewState, fRc, uLine)); return fRc; } #else DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState) { return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState); } #endif /** * Gets the state. */ DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState) { return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState); } #ifdef RTTIMER_LINUX_WITH_HRTIMER /** * Converts a nano second time stamp to ktime_t. * * ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts(). * * @returns ktime_t. * @param cNanoSecs Nanoseconds. */ DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs) { /* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */ return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000); } /** * Converts ktime_t to a nano second time stamp. * * ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts(). * * @returns nano second time stamp. * @param Kt ktime_t. */ DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt) { return ktime_to_ns(Kt); } #endif /* RTTIMER_LINUX_WITH_HRTIMER */ /** * Converts a nano second interval to jiffies. * * @returns Jiffies. * @param cNanoSecs Nanoseconds. */ DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs) { /* this can be made even better... */ if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET) return MAX_JIFFY_OFFSET; # if ARCH_BITS == 32 if (RT_LIKELY(cNanoSecs <= UINT32_MAX)) return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC; # endif return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC; } /** * Starts a sub-timer (RTTimerStart). * * @param pSubTimer The sub-timer to start. * @param u64Now The current timestamp (RTTimeSystemNanoTS()). * @param u64First The interval from u64Now to the first time the timer should fire. * @param fPinned true = timer pinned to a specific CPU, * false = timer can migrate between CPUs * @param fHighRes Whether the user requested a high resolution timer or not. * @param enmOldState The old timer state. */ static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First, bool fPinned, bool fHighRes) { /* * Calc when it should start firing. */ uint64_t u64NextTS = u64Now + u64First; if (!fHighRes) pSubTimer->u.Std.u64NextTS = u64NextTS; RTTIMERLNX_LOG(("startsubtimer %p\n", pSubTimer->pParent)); pSubTimer->iTick = 0; #ifdef RTTIMER_LINUX_WITH_HRTIMER if (fHighRes) hrtimer_start(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(u64NextTS), fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS); else #endif { unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First); pSubTimer->u.Std.ulNextJiffies = jiffies + cJiffies; pSubTimer->u.Std.fFirstAfterChg = true; #ifdef CONFIG_SMP if (fPinned) { # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); # else mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); # endif } else #endif mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); } /* Be a bit careful here since we could be racing the callback. */ if (!rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_STARTING)) rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_MP_STARTING); } /** * Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()). * * The caller has already changed the state, so we will not be in a callback * situation wrt to the calling thread. * * @param pSubTimer The sub-timer. * @param fHighRes Whether the user requested a high resolution timer or not. */ static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, bool fHighRes) { RTTIMERLNX_LOG(("stopsubtimer %p %d\n", pSubTimer->pParent, fHighRes)); #ifdef RTTIMER_LINUX_WITH_HRTIMER if (fHighRes) { /* There is no equivalent to del_timer in the hrtimer API, hrtimer_cancel() == del_timer_sync(). Just like the WARN_ON in del_timer_sync() asserts, waiting for a timer callback to complete is deadlock prone, so don't do it. */ int rc = hrtimer_try_to_cancel(&pSubTimer->u.Hr.LnxTimer); if (rc < 0) { hrtimer_start(&pSubTimer->u.Hr.LnxTimer, ktime_set(KTIME_SEC_MAX, 0), HRTIMER_MODE_ABS); hrtimer_try_to_cancel(&pSubTimer->u.Hr.LnxTimer); } } else #endif del_timer(&pSubTimer->u.Std.LnxTimer); rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); } /** * Used by RTTimerDestroy and rtTimerLnxCallbackDestroy to do the actual work. * * @param pTimer The timer in question. */ static void rtTimerLnxDestroyIt(PRTTIMER pTimer) { RTSPINLOCK hSpinlock = pTimer->hSpinlock; RTCPUID iCpu; Assert(pTimer->fSuspended); RTTIMERLNX_LOG(("destroyit %p\n", pTimer)); /* * Remove the MP notifications first because it'll reduce the risk of * us overtaking any MP event that might theoretically be racing us here. */ #ifdef CONFIG_SMP if ( pTimer->cCpus > 1 && hSpinlock != NIL_RTSPINLOCK) { int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer); AssertRC(rc); } #endif /* CONFIG_SMP */ /* * Invalidate the handle. */ ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC); /* * Make sure all timers have stopped executing since we're stopping them in * an asynchronous manner up in rtTimerLnxStopSubTimer. */ iCpu = pTimer->cCpus; while (iCpu-- > 0) { #ifdef RTTIMER_LINUX_WITH_HRTIMER if (pTimer->fHighRes) hrtimer_cancel(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer); else #endif del_timer_sync(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer); } /* * Finally, free the resources. */ RTMemFreeEx(pTimer, RT_UOFFSETOF_DYN(RTTIMER, aSubTimers[pTimer->cCpus])); if (hSpinlock != NIL_RTSPINLOCK) RTSpinlockDestroy(hSpinlock); } /** * Workqueue callback (no DECLCALLBACK!) for deferred destruction. * * @param pWork Pointer to the DtorWorkqueueItem member of our timer * structure. */ static void rtTimerLnxDestroyDeferred(RTR0LNXWORKQUEUEITEM *pWork) { PRTTIMER pTimer = RT_FROM_MEMBER(pWork, RTTIMER, DtorWorkqueueItem); rtTimerLnxDestroyIt(pTimer); } /** * Called when the timer was destroyed by the callback function. * * @param pTimer The timer. * @param pSubTimer The sub-timer which we're handling, the state of this * will be RTTIMERLNXSTATE_CALLBACK_DESTROYING. */ static void rtTimerLnxCallbackDestroy(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer) { /* * If it's an omni timer, the last dude does the destroying. */ if (pTimer->cCpus > 1) { uint32_t iCpu = pTimer->cCpus; RTSpinlockAcquire(pTimer->hSpinlock); Assert(pSubTimer->enmState == RTTIMERLNXSTATE_CB_DESTROYING); rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); while (iCpu-- > 0) if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED) { RTSpinlockRelease(pTimer->hSpinlock); return; } RTSpinlockRelease(pTimer->hSpinlock); } /* * Destroying a timer from the callback is unsafe since the callout code * might be touching the timer structure upon return (hrtimer does!). So, * we have to defer the actual destruction to the IRPT workqueue. */ rtR0LnxWorkqueuePush(&pTimer->DtorWorkqueueItem, rtTimerLnxDestroyDeferred); } #ifdef CONFIG_SMP /** * Deal with a sub-timer that has migrated. * * @param pTimer The timer. * @param pSubTimer The sub-timer. */ static void rtTimerLnxCallbackHandleMigration(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer) { RTTIMERLNXSTATE enmState; if (pTimer->cCpus > 1) RTSpinlockAcquire(pTimer->hSpinlock); do { enmState = rtTimerLnxGetState(&pSubTimer->enmState); switch (enmState) { case RTTIMERLNXSTATE_STOPPING: case RTTIMERLNXSTATE_MP_STOPPING: enmState = RTTIMERLNXSTATE_STOPPED; case RTTIMERLNXSTATE_STOPPED: break; default: AssertMsgFailed(("%d\n", enmState)); case RTTIMERLNXSTATE_STARTING: case RTTIMERLNXSTATE_MP_STARTING: case RTTIMERLNXSTATE_ACTIVE: case RTTIMERLNXSTATE_CALLBACK: case RTTIMERLNXSTATE_CB_STOPPING: case RTTIMERLNXSTATE_CB_RESTARTING: if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, enmState)) enmState = RTTIMERLNXSTATE_STOPPED; break; case RTTIMERLNXSTATE_CB_DESTROYING: { if (pTimer->cCpus > 1) RTSpinlockRelease(pTimer->hSpinlock); rtTimerLnxCallbackDestroy(pTimer, pSubTimer); return; } } } while (enmState != RTTIMERLNXSTATE_STOPPED); if (pTimer->cCpus > 1) RTSpinlockRelease(pTimer->hSpinlock); } #endif /* CONFIG_SMP */ /** * The slow path of rtTimerLnxChangeToCallbackState. * * @returns true if changed successfully, false if not. * @param pSubTimer The sub-timer. */ static bool rtTimerLnxChangeToCallbackStateSlow(PRTTIMERLNXSUBTIMER pSubTimer) { for (;;) { RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState); switch (enmState) { case RTTIMERLNXSTATE_ACTIVE: case RTTIMERLNXSTATE_STARTING: case RTTIMERLNXSTATE_MP_STARTING: if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, enmState)) return true; break; case RTTIMERLNXSTATE_CALLBACK: case RTTIMERLNXSTATE_CB_STOPPING: case RTTIMERLNXSTATE_CB_RESTARTING: case RTTIMERLNXSTATE_CB_DESTROYING: AssertMsgFailed(("%d\n", enmState)); default: return false; } ASMNopPause(); } } /** * Tries to change the sub-timer state to 'callback'. * * @returns true if changed successfully, false if not. * @param pSubTimer The sub-timer. */ DECLINLINE(bool) rtTimerLnxChangeToCallbackState(PRTTIMERLNXSUBTIMER pSubTimer) { if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, RTTIMERLNXSTATE_ACTIVE))) return true; return rtTimerLnxChangeToCallbackStateSlow(pSubTimer); } #ifdef RTTIMER_LINUX_WITH_HRTIMER /** * Timer callback function for high resolution timers. * * @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a * one-shot or interval timer. * @param pHrTimer Pointer to the sub-timer structure. */ static enum hrtimer_restart rtTimerLinuxHrCallback(struct hrtimer *pHrTimer) { PRTTIMERLNXSUBTIMER pSubTimer = RT_FROM_MEMBER(pHrTimer, RTTIMERLNXSUBTIMER, u.Hr.LnxTimer); PRTTIMER pTimer = pSubTimer->pParent; RTTIMERLNX_LOG(("hrcallback %p\n", pTimer)); if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer))) return HRTIMER_NORESTART; #ifdef CONFIG_SMP /* * Check for unwanted migration. */ if (pTimer->fAllCpus || pTimer->fSpecificCpu) { RTCPUID idCpu = RTMpCpuId(); if (RT_UNLIKELY( pTimer->fAllCpus ? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu : pTimer->idCpu != idCpu)) { rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer); return HRTIMER_NORESTART; } } #endif if (pTimer->u64NanoInterval) { /* * Periodic timer, run it and update the native timer afterwards so * we can handle RTTimerStop and RTTimerChangeInterval from the * callback as well as a racing control thread. */ pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick); hrtimer_add_expires_ns(&pSubTimer->u.Hr.LnxTimer, ASMAtomicReadU64(&pTimer->u64NanoInterval)); if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK))) return HRTIMER_RESTART; } else { /* * One shot timer (no omni), stop it before dispatching it. * Allow RTTimerStart as well as RTTimerDestroy to be called from * the callback. */ ASMAtomicWriteBool(&pTimer->fSuspended, true); pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick); if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK))) return HRTIMER_NORESTART; } /* * Some state change occurred while we were in the callback routine. */ for (;;) { RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState); switch (enmState) { case RTTIMERLNXSTATE_CB_DESTROYING: rtTimerLnxCallbackDestroy(pTimer, pSubTimer); return HRTIMER_NORESTART; case RTTIMERLNXSTATE_CB_STOPPING: if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING)) return HRTIMER_NORESTART; break; case RTTIMERLNXSTATE_CB_RESTARTING: if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING)) { pSubTimer->iTick = 0; hrtimer_set_expires(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(pSubTimer->uNsRestartAt)); return HRTIMER_RESTART; } break; default: AssertMsgFailed(("%d\n", enmState)); return HRTIMER_NORESTART; } ASMNopPause(); } } #endif /* RTTIMER_LINUX_WITH_HRTIMER */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0) /** * Timer callback function for standard timers. * * @param pLnxTimer Pointer to the Linux timer structure. */ static void rtTimerLinuxStdCallback(struct timer_list *pLnxTimer) { PRTTIMERLNXSUBTIMER pSubTimer = from_timer(pSubTimer, pLnxTimer, u.Std.LnxTimer); #else /** * Timer callback function for standard timers. * * @param ulUser Address of the sub-timer structure. */ static void rtTimerLinuxStdCallback(unsigned long ulUser) { PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser; #endif PRTTIMER pTimer = pSubTimer->pParent; RTTIMERLNX_LOG(("stdcallback %p\n", pTimer)); if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer))) return; #ifdef CONFIG_SMP /* * Check for unwanted migration. */ if (pTimer->fAllCpus || pTimer->fSpecificCpu) { RTCPUID idCpu = RTMpCpuId(); if (RT_UNLIKELY( pTimer->fAllCpus ? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu : pTimer->idCpu != idCpu)) { rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer); return; } } #endif if (pTimer->u64NanoInterval) { /* * Interval timer, calculate the next timeout. * * The first time around, we'll re-adjust the u.Std.u64NextTS to * try prevent some jittering if we were started at a bad time. */ const uint64_t iTick = ++pSubTimer->iTick; uint64_t u64NanoInterval; unsigned long cJiffies; unsigned long flFlags; spin_lock_irqsave(&pTimer->ChgIntLock, flFlags); u64NanoInterval = pTimer->u64NanoInterval; cJiffies = pTimer->cJiffies; if (RT_UNLIKELY(pSubTimer->u.Std.fFirstAfterChg)) { pSubTimer->u.Std.fFirstAfterChg = false; pSubTimer->u.Std.u64NextTS = RTTimeSystemNanoTS(); pSubTimer->u.Std.ulNextJiffies = jiffies; } spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags); pSubTimer->u.Std.u64NextTS += u64NanoInterval; if (cJiffies) { pSubTimer->u.Std.ulNextJiffies += cJiffies; /* Prevent overflows when the jiffies counter wraps around. * Special thanks to Ken Preslan for helping debugging! */ while (time_before(pSubTimer->u.Std.ulNextJiffies, jiffies)) { pSubTimer->u.Std.ulNextJiffies += cJiffies; pSubTimer->u.Std.u64NextTS += u64NanoInterval; } } else { const uint64_t u64NanoTS = RTTimeSystemNanoTS(); while (pSubTimer->u.Std.u64NextTS < u64NanoTS) pSubTimer->u.Std.u64NextTS += u64NanoInterval; pSubTimer->u.Std.ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u.Std.u64NextTS - u64NanoTS); } /* * Run the timer and re-arm it unless the state changed . * . * We must re-arm it afterwards as we're not in a position to undo this . * operation if for instance someone stopped or destroyed us while we . * were in the callback. (Linux takes care of any races here.) */ pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick); if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK))) { #ifdef CONFIG_SMP if (pTimer->fSpecificCpu || pTimer->fAllCpus) { # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); # else mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); # endif } else #endif mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); return; } } else { /* * One shot timer, stop it before dispatching it. * Allow RTTimerStart as well as RTTimerDestroy to be called from * the callback. */ ASMAtomicWriteBool(&pTimer->fSuspended, true); pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick); if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK))) return; } /* * Some state change occurred while we were in the callback routine. */ for (;;) { RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState); switch (enmState) { case RTTIMERLNXSTATE_CB_DESTROYING: rtTimerLnxCallbackDestroy(pTimer, pSubTimer); return; case RTTIMERLNXSTATE_CB_STOPPING: if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING)) return; break; case RTTIMERLNXSTATE_CB_RESTARTING: if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING)) { uint64_t u64NanoTS; uint64_t u64NextTS; unsigned long flFlags; spin_lock_irqsave(&pTimer->ChgIntLock, flFlags); u64NextTS = pSubTimer->uNsRestartAt; u64NanoTS = RTTimeSystemNanoTS(); pSubTimer->iTick = 0; pSubTimer->u.Std.u64NextTS = u64NextTS; pSubTimer->u.Std.fFirstAfterChg = true; pSubTimer->u.Std.ulNextJiffies = u64NextTS > u64NanoTS ? jiffies + rtTimerLnxNanoToJiffies(u64NextTS - u64NanoTS) : jiffies; spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags); #ifdef CONFIG_SMP if (pTimer->fSpecificCpu || pTimer->fAllCpus) { # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); # else mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); # endif } else #endif mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies); return; } break; default: AssertMsgFailed(("%d\n", enmState)); return; } ASMNopPause(); } } #ifdef CONFIG_SMP /** * Per-cpu callback function (RTMpOnAll/RTMpOnSpecific). * * @param idCpu The current CPU. * @param pvUser1 Pointer to the timer. * @param pvUser2 Pointer to the argument structure. */ static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2) { PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2; PRTTIMER pTimer = (PRTTIMER)pvUser1; Assert(idCpu < pTimer->cCpus); rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes); } /** * Worker for RTTimerStart() that takes care of the ugly bits. * * @returns RTTimerStart() return value. * @param pTimer The timer. * @param pArgs The argument structure. */ static int rtTimerLnxOmniStart(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs) { RTCPUID iCpu; RTCPUSET OnlineSet; RTCPUSET OnlineSet2; int rc2; /* * Prepare all the sub-timers for the startup and then flag the timer * as a whole as non-suspended, make sure we get them all before * clearing fSuspended as the MP handler will be waiting on this * should something happen while we're looping. */ RTSpinlockAcquire(pTimer->hSpinlock); /* Just make it a omni timer restriction that no stop/start races are allowed. */ for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++) if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED) { RTSpinlockRelease(pTimer->hSpinlock); return VERR_TIMER_BUSY; } do { RTMpGetOnlineSet(&OnlineSet); for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++) { Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING); rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState, RTCpuSetIsMember(&OnlineSet, iCpu) ? RTTIMERLNXSTATE_STARTING : RTTIMERLNXSTATE_STOPPED); } } while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2))); ASMAtomicWriteBool(&pTimer->fSuspended, false); RTSpinlockRelease(pTimer->hSpinlock); /* * Start them (can't find any exported function that allows me to * do this without the cross calls). */ pArgs->u64Now = RTTimeSystemNanoTS(); rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs); AssertRC(rc2); /* screw this if it fails. */ /* * Reset the sub-timers who didn't start up (ALL CPUs case). */ RTSpinlockAcquire(pTimer->hSpinlock); for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++) if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING)) { /** @todo very odd case for a rainy day. Cpus that temporarily went offline while * we were between calls needs to nudged as the MP handler will ignore events for * them because of the STARTING state. This is an extremely unlikely case - not that * that means anything in my experience... ;-) */ RTTIMERLNX_LOG(("what!? iCpu=%u -> didn't start\n", iCpu)); } RTSpinlockRelease(pTimer->hSpinlock); return VINF_SUCCESS; } /** * Worker for RTTimerStop() that takes care of the ugly SMP bits. * * @returns true if there was any active callbacks, false if not. * @param pTimer The timer (valid). * @param fForDestroy Whether this is for RTTimerDestroy or not. */ static bool rtTimerLnxOmniStop(PRTTIMER pTimer, bool fForDestroy) { bool fActiveCallbacks = false; RTCPUID iCpu; RTTIMERLNXSTATE enmState; /* * Mark the timer as suspended and flag all timers as stopping, except * for those being stopped by an MP event. */ RTSpinlockAcquire(pTimer->hSpinlock); ASMAtomicWriteBool(&pTimer->fSuspended, true); for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++) { for (;;) { enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState); if ( enmState == RTTIMERLNXSTATE_STOPPED || enmState == RTTIMERLNXSTATE_MP_STOPPING) break; if ( enmState == RTTIMERLNXSTATE_CALLBACK || enmState == RTTIMERLNXSTATE_CB_STOPPING || enmState == RTTIMERLNXSTATE_CB_RESTARTING) { Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING || fForDestroy); if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, !fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING, enmState)) { fActiveCallbacks = true; break; } } else { Assert(enmState == RTTIMERLNXSTATE_ACTIVE); if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState)) break; } ASMNopPause(); } } RTSpinlockRelease(pTimer->hSpinlock); /* * Do the actual stopping. Fortunately, this doesn't require any IPIs. * Unfortunately it cannot be done synchronously. */ for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++) if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING) rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu], pTimer->fHighRes); return fActiveCallbacks; } /** * Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent() * to start a sub-timer on a cpu that just have come online. * * @param idCpu The current CPU. * @param pvUser1 Pointer to the timer. * @param pvUser2 Pointer to the argument structure. */ static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2) { PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2; PRTTIMER pTimer = (PRTTIMER)pvUser1; RTSPINLOCK hSpinlock; Assert(idCpu < pTimer->cCpus); /* * We have to be kind of careful here as we might be racing RTTimerStop * (and/or RTTimerDestroy, thus the paranoia. */ hSpinlock = pTimer->hSpinlock; if ( hSpinlock != NIL_RTSPINLOCK && pTimer->u32Magic == RTTIMER_MAGIC) { RTSpinlockAcquire(hSpinlock); if ( !ASMAtomicUoReadBool(&pTimer->fSuspended) && pTimer->u32Magic == RTTIMER_MAGIC) { /* We're sane and the timer is not suspended yet. */ PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu]; if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED)) rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes); } RTSpinlockRelease(hSpinlock); } } /** * MP event notification callback. * * @param enmEvent The event. * @param idCpu The cpu it applies to. * @param pvUser The timer. */ static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser) { PRTTIMER pTimer = (PRTTIMER)pvUser; PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu]; RTSPINLOCK hSpinlock; Assert(idCpu < pTimer->cCpus); /* * Some initial paranoia. */ if (pTimer->u32Magic != RTTIMER_MAGIC) return; hSpinlock = pTimer->hSpinlock; if (hSpinlock == NIL_RTSPINLOCK) return; RTSpinlockAcquire(hSpinlock); /* Is it active? */ if ( !ASMAtomicUoReadBool(&pTimer->fSuspended) && pTimer->u32Magic == RTTIMER_MAGIC) { switch (enmEvent) { /* * Try do it without leaving the spin lock, but if we have to, retake it * when we're on the right cpu. */ case RTMPEVENT_ONLINE: if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED)) { RTTIMERLINUXSTARTONCPUARGS Args; Args.u64Now = RTTimeSystemNanoTS(); Args.u64First = 0; if (RTMpCpuId() == idCpu) rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /*fPinned*/, pTimer->fHighRes); else { rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */ RTSpinlockRelease(hSpinlock); RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args); return; /* we've left the spinlock */ } } break; /* * The CPU is (going) offline, make sure the sub-timer is stopped. * * Linux will migrate it to a different CPU, but we don't want this. The * timer function is checking for this. */ case RTMPEVENT_OFFLINE: { RTTIMERLNXSTATE enmState; while ( (enmState = rtTimerLnxGetState(&pSubTimer->enmState)) == RTTIMERLNXSTATE_ACTIVE || enmState == RTTIMERLNXSTATE_CALLBACK || enmState == RTTIMERLNXSTATE_CB_RESTARTING) { if (enmState == RTTIMERLNXSTATE_ACTIVE) { if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE)) { RTSpinlockRelease(hSpinlock); rtTimerLnxStopSubTimer(pSubTimer, pTimer->fHighRes); return; /* we've left the spinlock */ } } else if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CB_STOPPING, enmState)) break; /* State not stable, try again. */ ASMNopPause(); } break; } } } RTSpinlockRelease(hSpinlock); } #endif /* CONFIG_SMP */ /** * Callback function use by RTTimerStart via RTMpOnSpecific to start a timer * running on a specific CPU. * * @param idCpu The current CPU. * @param pvUser1 Pointer to the timer. * @param pvUser2 Pointer to the argument structure. */ static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2) { PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2; PRTTIMER pTimer = (PRTTIMER)pvUser1; RT_NOREF_PV(idCpu); rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /*fPinned*/, pTimer->fHighRes); } RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First) { RTTIMERLINUXSTARTONCPUARGS Args; int rc2; IPRT_LINUX_SAVE_EFL_AC(); /* * Validate. */ AssertPtrReturn(pTimer, VERR_INVALID_HANDLE); AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE); if (!ASMAtomicUoReadBool(&pTimer->fSuspended)) return VERR_TIMER_ACTIVE; RTTIMERLNX_LOG(("start %p cCpus=%d\n", pTimer, pTimer->cCpus)); Args.u64First = u64First; #ifdef CONFIG_SMP /* * Omni timer? */ if (pTimer->fAllCpus) { rc2 = rtTimerLnxOmniStart(pTimer, &Args); IPRT_LINUX_RESTORE_EFL_AC(); return rc2; } #endif /* * Simple timer - Pretty straight forward if it wasn't for restarting. */ Args.u64Now = RTTimeSystemNanoTS(); ASMAtomicWriteU64(&pTimer->aSubTimers[0].uNsRestartAt, Args.u64Now + u64First); for (;;) { RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState); switch (enmState) { case RTTIMERLNXSTATE_STOPPED: if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING, RTTIMERLNXSTATE_STOPPED)) { ASMAtomicWriteBool(&pTimer->fSuspended, false); if (!pTimer->fSpecificCpu) rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First, false /*fPinned*/, pTimer->fHighRes); else { rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args); if (RT_FAILURE(rc2)) { /* Suspend it, the cpu id is probably invalid or offline. */ ASMAtomicWriteBool(&pTimer->fSuspended, true); rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED); return rc2; } } IPRT_LINUX_RESTORE_EFL_AC(); return VINF_SUCCESS; } break; case RTTIMERLNXSTATE_CALLBACK: case RTTIMERLNXSTATE_CB_STOPPING: if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_CB_RESTARTING, enmState)) { ASMAtomicWriteBool(&pTimer->fSuspended, false); IPRT_LINUX_RESTORE_EFL_AC(); return VINF_SUCCESS; } break; default: AssertMsgFailed(("%d\n", enmState)); IPRT_LINUX_RESTORE_EFL_AC(); return VERR_INTERNAL_ERROR_4; } ASMNopPause(); } } RT_EXPORT_SYMBOL(RTTimerStart); /** * Common worker for RTTimerStop and RTTimerDestroy. * * @returns true if there was any active callbacks, false if not. * @param pTimer The timer to stop. * @param fForDestroy Whether it's RTTimerDestroy calling or not. */ static bool rtTimerLnxStop(PRTTIMER pTimer, bool fForDestroy) { RTTIMERLNX_LOG(("lnxstop %p %d\n", pTimer, fForDestroy)); #ifdef CONFIG_SMP /* * Omni timer? */ if (pTimer->fAllCpus) return rtTimerLnxOmniStop(pTimer, fForDestroy); #endif /* * Simple timer. */ ASMAtomicWriteBool(&pTimer->fSuspended, true); for (;;) { RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState); switch (enmState) { case RTTIMERLNXSTATE_ACTIVE: if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING, RTTIMERLNXSTATE_ACTIVE)) { rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0], pTimer->fHighRes); return false; } break; case RTTIMERLNXSTATE_CALLBACK: case RTTIMERLNXSTATE_CB_RESTARTING: case RTTIMERLNXSTATE_CB_STOPPING: Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING || fForDestroy); if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, !fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING, enmState)) return true; break; case RTTIMERLNXSTATE_STOPPED: return VINF_SUCCESS; case RTTIMERLNXSTATE_CB_DESTROYING: AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer)); return true; default: case RTTIMERLNXSTATE_STARTING: case RTTIMERLNXSTATE_MP_STARTING: case RTTIMERLNXSTATE_STOPPING: case RTTIMERLNXSTATE_MP_STOPPING: AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer)); return false; } /* State not stable, try again. */ ASMNopPause(); } } RTDECL(int) RTTimerStop(PRTTIMER pTimer) { /* * Validate. */ IPRT_LINUX_SAVE_EFL_AC(); AssertPtrReturn(pTimer, VERR_INVALID_HANDLE); AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE); RTTIMERLNX_LOG(("stop %p\n", pTimer)); if (ASMAtomicUoReadBool(&pTimer->fSuspended)) return VERR_TIMER_SUSPENDED; rtTimerLnxStop(pTimer, false /*fForDestroy*/); IPRT_LINUX_RESTORE_EFL_AC(); return VINF_SUCCESS; } RT_EXPORT_SYMBOL(RTTimerStop); RTDECL(int) RTTimerChangeInterval(PRTTIMER pTimer, uint64_t u64NanoInterval) { unsigned long cJiffies; unsigned long flFlags; IPRT_LINUX_SAVE_EFL_AC(); /* * Validate. */ AssertPtrReturn(pTimer, VERR_INVALID_HANDLE); AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE); AssertReturn(u64NanoInterval, VERR_INVALID_PARAMETER); AssertReturn(u64NanoInterval < UINT64_MAX / 8, VERR_INVALID_PARAMETER); AssertReturn(pTimer->u64NanoInterval, VERR_INVALID_STATE); RTTIMERLNX_LOG(("change %p %llu\n", pTimer, u64NanoInterval)); #ifdef RTTIMER_LINUX_WITH_HRTIMER /* * For the high resolution timers it is easy since we don't care so much * about when it is applied to the sub-timers. */ if (pTimer->fHighRes) { ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval); IPRT_LINUX_RESTORE_EFL_AC(); return VINF_SUCCESS; } #endif /* * Standard timers have a bit more complicated way of calculating * their interval and such. So, forget omni timers for now. */ if (pTimer->cCpus > 1) return VERR_NOT_SUPPORTED; cJiffies = u64NanoInterval / RTTimerGetSystemGranularity(); if (cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval) cJiffies = 0; spin_lock_irqsave(&pTimer->ChgIntLock, flFlags); pTimer->aSubTimers[0].u.Std.fFirstAfterChg = true; pTimer->cJiffies = cJiffies; ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval); spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags); IPRT_LINUX_RESTORE_EFL_AC(); return VINF_SUCCESS; } RT_EXPORT_SYMBOL(RTTimerChangeInterval); RTDECL(int) RTTimerDestroy(PRTTIMER pTimer) { bool fCanDestroy; IPRT_LINUX_SAVE_EFL_AC(); /* * Validate. It's ok to pass NULL pointer. */ if (pTimer == /*NIL_RTTIMER*/ NULL) return VINF_SUCCESS; AssertPtrReturn(pTimer, VERR_INVALID_HANDLE); AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE); RTTIMERLNX_LOG(("destroy %p\n", pTimer)); /** @todo We should invalidate the magic here! */ /* * Stop the timer if it's still active, then destroy it if we can. */ if (!ASMAtomicUoReadBool(&pTimer->fSuspended)) fCanDestroy = rtTimerLnxStop(pTimer, true /*fForDestroy*/); else { uint32_t iCpu = pTimer->cCpus; if (pTimer->cCpus > 1) RTSpinlockAcquire(pTimer->hSpinlock); fCanDestroy = true; while (iCpu-- > 0) { for (;;) { RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState); switch (enmState) { case RTTIMERLNXSTATE_CALLBACK: case RTTIMERLNXSTATE_CB_RESTARTING: case RTTIMERLNXSTATE_CB_STOPPING: if (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_CB_DESTROYING, enmState)) continue; fCanDestroy = false; break; case RTTIMERLNXSTATE_CB_DESTROYING: AssertMsgFailed(("%d\n", enmState)); fCanDestroy = false; break; default: break; } break; } } if (pTimer->cCpus > 1) RTSpinlockRelease(pTimer->hSpinlock); } if (fCanDestroy) { /* For paranoid reasons, defer actually destroying the semaphore when in atomic or interrupt context. */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 32) if (in_atomic() || in_interrupt()) #else if (in_interrupt()) #endif rtR0LnxWorkqueuePush(&pTimer->DtorWorkqueueItem, rtTimerLnxDestroyDeferred); else rtTimerLnxDestroyIt(pTimer); } IPRT_LINUX_RESTORE_EFL_AC(); return VINF_SUCCESS; } RT_EXPORT_SYMBOL(RTTimerDestroy); RTDECL(int) RTTimerCreateEx(PRTTIMER *ppTimer, uint64_t u64NanoInterval, uint32_t fFlags, PFNRTTIMER pfnTimer, void *pvUser) { PRTTIMER pTimer; RTCPUID iCpu; unsigned cCpus; int rc; IPRT_LINUX_SAVE_EFL_AC(); rtR0LnxWorkqueueFlush(); /* for 2.4 */ *ppTimer = NULL; /* * Validate flags. */ if (!RTTIMER_FLAGS_ARE_VALID(fFlags)) { IPRT_LINUX_RESTORE_EFL_AC(); return VERR_INVALID_PARAMETER; } if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL && !RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK))) { IPRT_LINUX_RESTORE_EFL_AC(); return VERR_CPU_NOT_FOUND; } /* * Allocate the timer handler. */ cCpus = 1; #ifdef CONFIG_SMP if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL) { cCpus = RTMpGetMaxCpuId() + 1; Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */ AssertReturnStmt(u64NanoInterval, IPRT_LINUX_RESTORE_EFL_AC(), VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */ } #endif rc = RTMemAllocEx(RT_UOFFSETOF_DYN(RTTIMER, aSubTimers[cCpus]), 0, RTMEMALLOCEX_FLAGS_ZEROED | RTMEMALLOCEX_FLAGS_ANY_CTX_FREE, (void **)&pTimer); if (RT_FAILURE(rc)) { IPRT_LINUX_RESTORE_EFL_AC(); return rc; } /* * Initialize it. */ pTimer->u32Magic = RTTIMER_MAGIC; pTimer->hSpinlock = NIL_RTSPINLOCK; pTimer->fSuspended = true; pTimer->fHighRes = !!(fFlags & RTTIMER_FLAGS_HIGH_RES); #ifdef CONFIG_SMP pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL; pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL; pTimer->idCpu = pTimer->fSpecificCpu ? RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK) : NIL_RTCPUID; #else pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC); pTimer->idCpu = RTMpCpuId(); #endif pTimer->cCpus = cCpus; pTimer->pfnTimer = pfnTimer; pTimer->pvUser = pvUser; pTimer->u64NanoInterval = u64NanoInterval; pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity(); if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval) pTimer->cJiffies = 0; spin_lock_init(&pTimer->ChgIntLock); for (iCpu = 0; iCpu < cCpus; iCpu++) { #ifdef RTTIMER_LINUX_WITH_HRTIMER if (pTimer->fHighRes) { hrtimer_init(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); pTimer->aSubTimers[iCpu].u.Hr.LnxTimer.function = rtTimerLinuxHrCallback; } else #endif { #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0) timer_setup(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer, rtTimerLinuxStdCallback, TIMER_PINNED); #elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) init_timer_pinned(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer); #else init_timer(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer); #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) pTimer->aSubTimers[iCpu].u.Std.LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu]; pTimer->aSubTimers[iCpu].u.Std.LnxTimer.function = rtTimerLinuxStdCallback; #endif pTimer->aSubTimers[iCpu].u.Std.LnxTimer.expires = jiffies; pTimer->aSubTimers[iCpu].u.Std.u64NextTS = 0; } pTimer->aSubTimers[iCpu].iTick = 0; pTimer->aSubTimers[iCpu].pParent = pTimer; pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED; } #ifdef CONFIG_SMP /* * If this is running on ALL cpus, we'll have to register a callback * for MP events (so timers can be started/stopped on cpus going * online/offline). We also create the spinlock for synchronizing * stop/start/mp-event. */ if (cCpus > 1) { int rc = RTSpinlockCreate(&pTimer->hSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_SAFE, "RTTimerLnx"); if (RT_SUCCESS(rc)) rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer); else pTimer->hSpinlock = NIL_RTSPINLOCK; if (RT_FAILURE(rc)) { RTTimerDestroy(pTimer); IPRT_LINUX_RESTORE_EFL_AC(); return rc; } } #endif /* CONFIG_SMP */ RTTIMERLNX_LOG(("create %p hires=%d fFlags=%#x cCpus=%u\n", pTimer, pTimer->fHighRes, fFlags, cCpus)); *ppTimer = pTimer; IPRT_LINUX_RESTORE_EFL_AC(); return VINF_SUCCESS; } RT_EXPORT_SYMBOL(RTTimerCreateEx); RTDECL(uint32_t) RTTimerGetSystemGranularity(void) { #if 0 /** @todo Not sure if this is what we want or not... Add new API for * querying the resolution of the high res timers? */ struct timespec Ts; int rc; IPRT_LINUX_SAVE_EFL_AC(); rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts); IPRT_LINUX_RESTORE_EFL_AC(); if (!rc) { Assert(!Ts.tv_sec); return Ts.tv_nsec; } #endif return RT_NS_1SEC / HZ; /* ns */ } RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity); RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted) { RT_NOREF_PV(u32Request); RT_NOREF_PV(*pu32Granted); return VERR_NOT_SUPPORTED; } RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity); RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted) { RT_NOREF_PV(u32Granted); return VERR_NOT_SUPPORTED; } RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity); RTDECL(bool) RTTimerCanDoHighResolution(void) { #ifdef RTTIMER_LINUX_WITH_HRTIMER return true; #else return false; #endif } RT_EXPORT_SYMBOL(RTTimerCanDoHighResolution);