From ace9429bb58fd418f0c81d4c2835699bddf6bde6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 11 Apr 2024 10:27:49 +0200 Subject: Adding upstream version 6.6.15. Signed-off-by: Daniel Baumann --- drivers/cpuidle/governors/Makefile | 9 + drivers/cpuidle/governors/gov.h | 14 + drivers/cpuidle/governors/haltpoll.c | 152 ++++++++ drivers/cpuidle/governors/ladder.c | 197 ++++++++++ drivers/cpuidle/governors/menu.c | 590 +++++++++++++++++++++++++++++ drivers/cpuidle/governors/teo.c | 695 +++++++++++++++++++++++++++++++++++ 6 files changed, 1657 insertions(+) create mode 100644 drivers/cpuidle/governors/Makefile create mode 100644 drivers/cpuidle/governors/gov.h create mode 100644 drivers/cpuidle/governors/haltpoll.c create mode 100644 drivers/cpuidle/governors/ladder.c create mode 100644 drivers/cpuidle/governors/menu.c create mode 100644 drivers/cpuidle/governors/teo.c (limited to 'drivers/cpuidle/governors') diff --git a/drivers/cpuidle/governors/Makefile b/drivers/cpuidle/governors/Makefile new file mode 100644 index 0000000000..63abb5393a --- /dev/null +++ b/drivers/cpuidle/governors/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for cpuidle governors. +# + +obj-$(CONFIG_CPU_IDLE_GOV_LADDER) += ladder.o +obj-$(CONFIG_CPU_IDLE_GOV_MENU) += menu.o +obj-$(CONFIG_CPU_IDLE_GOV_TEO) += teo.o +obj-$(CONFIG_CPU_IDLE_GOV_HALTPOLL) += haltpoll.o diff --git a/drivers/cpuidle/governors/gov.h b/drivers/cpuidle/governors/gov.h new file mode 100644 index 0000000000..99e067d966 --- /dev/null +++ b/drivers/cpuidle/governors/gov.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Common definitions for cpuidle governors. */ + +#ifndef __CPUIDLE_GOVERNOR_H +#define __CPUIDLE_GOVERNOR_H + +/* + * Idle state target residency threshold used for deciding whether or not to + * check the time till the closest expected timer event. + */ +#define RESIDENCY_THRESHOLD_NS (15 * NSEC_PER_USEC) + +#endif /* __CPUIDLE_GOVERNOR_H */ diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c new file mode 100644 index 0000000000..1dff3a5291 --- /dev/null +++ b/drivers/cpuidle/governors/haltpoll.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * haltpoll.c - haltpoll idle governor + * + * Copyright 2019 Red Hat, Inc. and/or its affiliates. + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Authors: Marcelo Tosatti + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned int guest_halt_poll_ns __read_mostly = 200000; +module_param(guest_halt_poll_ns, uint, 0644); + +/* division factor to shrink halt_poll_ns */ +static unsigned int guest_halt_poll_shrink __read_mostly = 2; +module_param(guest_halt_poll_shrink, uint, 0644); + +/* multiplication factor to grow per-cpu poll_limit_ns */ +static unsigned int guest_halt_poll_grow __read_mostly = 2; +module_param(guest_halt_poll_grow, uint, 0644); + +/* value in us to start growing per-cpu halt_poll_ns */ +static unsigned int guest_halt_poll_grow_start __read_mostly = 50000; +module_param(guest_halt_poll_grow_start, uint, 0644); + +/* allow shrinking guest halt poll */ +static bool guest_halt_poll_allow_shrink __read_mostly = true; +module_param(guest_halt_poll_allow_shrink, bool, 0644); + +/** + * haltpoll_select - selects the next idle state to enter + * @drv: cpuidle driver containing state data + * @dev: the CPU + * @stop_tick: indication on whether or not to stop the tick + */ +static int haltpoll_select(struct cpuidle_driver *drv, + struct cpuidle_device *dev, + bool *stop_tick) +{ + s64 latency_req = cpuidle_governor_latency_req(dev->cpu); + + if (!drv->state_count || latency_req == 0) { + *stop_tick = false; + return 0; + } + + if (dev->poll_limit_ns == 0) + return 1; + + /* Last state was poll? */ + if (dev->last_state_idx == 0) { + /* Halt if no event occurred on poll window */ + if (dev->poll_time_limit == true) + return 1; + + *stop_tick = false; + /* Otherwise, poll again */ + return 0; + } + + *stop_tick = false; + /* Last state was halt: poll */ + return 0; +} + +static void adjust_poll_limit(struct cpuidle_device *dev, u64 block_ns) +{ + unsigned int val; + + /* Grow cpu_halt_poll_us if + * cpu_halt_poll_us < block_ns < guest_halt_poll_us + */ + if (block_ns > dev->poll_limit_ns && block_ns <= guest_halt_poll_ns) { + val = dev->poll_limit_ns * guest_halt_poll_grow; + + if (val < guest_halt_poll_grow_start) + val = guest_halt_poll_grow_start; + if (val > guest_halt_poll_ns) + val = guest_halt_poll_ns; + + trace_guest_halt_poll_ns_grow(val, dev->poll_limit_ns); + dev->poll_limit_ns = val; + } else if (block_ns > guest_halt_poll_ns && + guest_halt_poll_allow_shrink) { + unsigned int shrink = guest_halt_poll_shrink; + + val = dev->poll_limit_ns; + if (shrink == 0) + val = 0; + else + val /= shrink; + trace_guest_halt_poll_ns_shrink(val, dev->poll_limit_ns); + dev->poll_limit_ns = val; + } +} + +/** + * haltpoll_reflect - update variables and update poll time + * @dev: the CPU + * @index: the index of actual entered state + */ +static void haltpoll_reflect(struct cpuidle_device *dev, int index) +{ + dev->last_state_idx = index; + + if (index != 0) + adjust_poll_limit(dev, dev->last_residency_ns); +} + +/** + * haltpoll_enable_device - scans a CPU's states and does setup + * @drv: cpuidle driver + * @dev: the CPU + */ +static int haltpoll_enable_device(struct cpuidle_driver *drv, + struct cpuidle_device *dev) +{ + dev->poll_limit_ns = 0; + + return 0; +} + +static struct cpuidle_governor haltpoll_governor = { + .name = "haltpoll", + .rating = 9, + .enable = haltpoll_enable_device, + .select = haltpoll_select, + .reflect = haltpoll_reflect, +}; + +static int __init init_haltpoll(void) +{ + if (kvm_para_available()) + return cpuidle_register_governor(&haltpoll_governor); + + return 0; +} + +postcore_initcall(init_haltpoll); diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c new file mode 100644 index 0000000000..8e9058c4ea --- /dev/null +++ b/drivers/cpuidle/governors/ladder.c @@ -0,0 +1,197 @@ +/* + * ladder.c - the residency ladder algorithm + * + * Copyright (C) 2001, 2002 Andy Grover + * Copyright (C) 2001, 2002 Paul Diefenbaugh + * Copyright (C) 2004, 2005 Dominik Brodowski + * + * (C) 2006-2007 Venkatesh Pallipadi + * Shaohua Li + * Adam Belay + * + * This code is licenced under the GPL. + */ + +#include +#include +#include +#include + +#include +#include + +#define PROMOTION_COUNT 4 +#define DEMOTION_COUNT 1 + +struct ladder_device_state { + struct { + u32 promotion_count; + u32 demotion_count; + u64 promotion_time_ns; + u64 demotion_time_ns; + } threshold; + struct { + int promotion_count; + int demotion_count; + } stats; +}; + +struct ladder_device { + struct ladder_device_state states[CPUIDLE_STATE_MAX]; +}; + +static DEFINE_PER_CPU(struct ladder_device, ladder_devices); + +/** + * ladder_do_selection - prepares private data for a state change + * @ldev: the ladder device + * @old_idx: the current state index + * @new_idx: the new target state index + */ +static inline void ladder_do_selection(struct cpuidle_device *dev, + struct ladder_device *ldev, + int old_idx, int new_idx) +{ + ldev->states[old_idx].stats.promotion_count = 0; + ldev->states[old_idx].stats.demotion_count = 0; + dev->last_state_idx = new_idx; +} + +/** + * ladder_select_state - selects the next state to enter + * @drv: cpuidle driver + * @dev: the CPU + * @dummy: not used + */ +static int ladder_select_state(struct cpuidle_driver *drv, + struct cpuidle_device *dev, bool *dummy) +{ + struct ladder_device *ldev = this_cpu_ptr(&ladder_devices); + struct ladder_device_state *last_state; + int last_idx = dev->last_state_idx; + int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0; + s64 latency_req = cpuidle_governor_latency_req(dev->cpu); + s64 last_residency; + + /* Special case when user has set very strict latency requirement */ + if (unlikely(latency_req == 0)) { + ladder_do_selection(dev, ldev, last_idx, 0); + return 0; + } + + last_state = &ldev->states[last_idx]; + + last_residency = dev->last_residency_ns - drv->states[last_idx].exit_latency_ns; + + /* consider promotion */ + if (last_idx < drv->state_count - 1 && + !dev->states_usage[last_idx + 1].disable && + last_residency > last_state->threshold.promotion_time_ns && + drv->states[last_idx + 1].exit_latency_ns <= latency_req) { + last_state->stats.promotion_count++; + last_state->stats.demotion_count = 0; + if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) { + ladder_do_selection(dev, ldev, last_idx, last_idx + 1); + return last_idx + 1; + } + } + + /* consider demotion */ + if (last_idx > first_idx && + (dev->states_usage[last_idx].disable || + drv->states[last_idx].exit_latency_ns > latency_req)) { + int i; + + for (i = last_idx - 1; i > first_idx; i--) { + if (drv->states[i].exit_latency_ns <= latency_req) + break; + } + ladder_do_selection(dev, ldev, last_idx, i); + return i; + } + + if (last_idx > first_idx && + last_residency < last_state->threshold.demotion_time_ns) { + last_state->stats.demotion_count++; + last_state->stats.promotion_count = 0; + if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) { + ladder_do_selection(dev, ldev, last_idx, last_idx - 1); + return last_idx - 1; + } + } + + /* otherwise remain at the current state */ + return last_idx; +} + +/** + * ladder_enable_device - setup for the governor + * @drv: cpuidle driver + * @dev: the CPU + */ +static int ladder_enable_device(struct cpuidle_driver *drv, + struct cpuidle_device *dev) +{ + int i; + int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0; + struct ladder_device *ldev = &per_cpu(ladder_devices, dev->cpu); + struct ladder_device_state *lstate; + struct cpuidle_state *state; + + dev->last_state_idx = first_idx; + + for (i = first_idx; i < drv->state_count; i++) { + state = &drv->states[i]; + lstate = &ldev->states[i]; + + lstate->stats.promotion_count = 0; + lstate->stats.demotion_count = 0; + + lstate->threshold.promotion_count = PROMOTION_COUNT; + lstate->threshold.demotion_count = DEMOTION_COUNT; + + if (i < drv->state_count - 1) + lstate->threshold.promotion_time_ns = state->exit_latency_ns; + if (i > first_idx) + lstate->threshold.demotion_time_ns = state->exit_latency_ns; + } + + return 0; +} + +/** + * ladder_reflect - update the correct last_state_idx + * @dev: the CPU + * @index: the index of actual state entered + */ +static void ladder_reflect(struct cpuidle_device *dev, int index) +{ + if (index > 0) + dev->last_state_idx = index; +} + +static struct cpuidle_governor ladder_governor = { + .name = "ladder", + .rating = 10, + .enable = ladder_enable_device, + .select = ladder_select_state, + .reflect = ladder_reflect, +}; + +/** + * init_ladder - initializes the governor + */ +static int __init init_ladder(void) +{ + /* + * When NO_HZ is disabled, or when booting with nohz=off, the ladder + * governor is better so give it a higher rating than the menu + * governor. + */ + if (!tick_nohz_enabled) + ladder_governor.rating = 25; + + return cpuidle_register_governor(&ladder_governor); +} + +postcore_initcall(init_ladder); diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c new file mode 100644 index 0000000000..b96e3da0fe --- /dev/null +++ b/drivers/cpuidle/governors/menu.c @@ -0,0 +1,590 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * menu.c - the menu idle governor + * + * Copyright (C) 2006-2007 Adam Belay + * Copyright (C) 2009 Intel Corporation + * Author: + * Arjan van de Ven + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gov.h" + +#define BUCKETS 12 +#define INTERVAL_SHIFT 3 +#define INTERVALS (1UL << INTERVAL_SHIFT) +#define RESOLUTION 1024 +#define DECAY 8 +#define MAX_INTERESTING (50000 * NSEC_PER_USEC) + +/* + * Concepts and ideas behind the menu governor + * + * For the menu governor, there are 3 decision factors for picking a C + * state: + * 1) Energy break even point + * 2) Performance impact + * 3) Latency tolerance (from pmqos infrastructure) + * These three factors are treated independently. + * + * Energy break even point + * ----------------------- + * C state entry and exit have an energy cost, and a certain amount of time in + * the C state is required to actually break even on this cost. CPUIDLE + * provides us this duration in the "target_residency" field. So all that we + * need is a good prediction of how long we'll be idle. Like the traditional + * menu governor, we start with the actual known "next timer event" time. + * + * Since there are other source of wakeups (interrupts for example) than + * the next timer event, this estimation is rather optimistic. To get a + * more realistic estimate, a correction factor is applied to the estimate, + * that is based on historic behavior. For example, if in the past the actual + * duration always was 50% of the next timer tick, the correction factor will + * be 0.5. + * + * menu uses a running average for this correction factor, however it uses a + * set of factors, not just a single factor. This stems from the realization + * that the ratio is dependent on the order of magnitude of the expected + * duration; if we expect 500 milliseconds of idle time the likelihood of + * getting an interrupt very early is much higher than if we expect 50 micro + * seconds of idle time. A second independent factor that has big impact on + * the actual factor is if there is (disk) IO outstanding or not. + * (as a special twist, we consider every sleep longer than 50 milliseconds + * as perfect; there are no power gains for sleeping longer than this) + * + * For these two reasons we keep an array of 12 independent factors, that gets + * indexed based on the magnitude of the expected duration as well as the + * "is IO outstanding" property. + * + * Repeatable-interval-detector + * ---------------------------- + * There are some cases where "next timer" is a completely unusable predictor: + * Those cases where the interval is fixed, for example due to hardware + * interrupt mitigation, but also due to fixed transfer rate devices such as + * mice. + * For this, we use a different predictor: We track the duration of the last 8 + * intervals and if the stand deviation of these 8 intervals is below a + * threshold value, we use the average of these intervals as prediction. + * + * Limiting Performance Impact + * --------------------------- + * C states, especially those with large exit latencies, can have a real + * noticeable impact on workloads, which is not acceptable for most sysadmins, + * and in addition, less performance has a power price of its own. + * + * As a general rule of thumb, menu assumes that the following heuristic + * holds: + * The busier the system, the less impact of C states is acceptable + * + * This rule-of-thumb is implemented using a performance-multiplier: + * If the exit latency times the performance multiplier is longer than + * the predicted duration, the C state is not considered a candidate + * for selection due to a too high performance impact. So the higher + * this multiplier is, the longer we need to be idle to pick a deep C + * state, and thus the less likely a busy CPU will hit such a deep + * C state. + * + * Two factors are used in determing this multiplier: + * a value of 10 is added for each point of "per cpu load average" we have. + * a value of 5 points is added for each process that is waiting for + * IO on this CPU. + * (these values are experimentally determined) + * + * The load average factor gives a longer term (few seconds) input to the + * decision, while the iowait value gives a cpu local instantanious input. + * The iowait factor may look low, but realize that this is also already + * represented in the system load average. + * + */ + +struct menu_device { + int needs_update; + int tick_wakeup; + + u64 next_timer_ns; + unsigned int bucket; + unsigned int correction_factor[BUCKETS]; + unsigned int intervals[INTERVALS]; + int interval_ptr; +}; + +static inline int which_bucket(u64 duration_ns, unsigned int nr_iowaiters) +{ + int bucket = 0; + + /* + * We keep two groups of stats; one with no + * IO pending, one without. + * This allows us to calculate + * E(duration)|iowait + */ + if (nr_iowaiters) + bucket = BUCKETS/2; + + if (duration_ns < 10ULL * NSEC_PER_USEC) + return bucket; + if (duration_ns < 100ULL * NSEC_PER_USEC) + return bucket + 1; + if (duration_ns < 1000ULL * NSEC_PER_USEC) + return bucket + 2; + if (duration_ns < 10000ULL * NSEC_PER_USEC) + return bucket + 3; + if (duration_ns < 100000ULL * NSEC_PER_USEC) + return bucket + 4; + return bucket + 5; +} + +/* + * Return a multiplier for the exit latency that is intended + * to take performance requirements into account. + * The more performance critical we estimate the system + * to be, the higher this multiplier, and thus the higher + * the barrier to go to an expensive C state. + */ +static inline int performance_multiplier(unsigned int nr_iowaiters) +{ + /* for IO wait tasks (per cpu!) we add 10x each */ + return 1 + 10 * nr_iowaiters; +} + +static DEFINE_PER_CPU(struct menu_device, menu_devices); + +static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev); + +/* + * Try detecting repeating patterns by keeping track of the last 8 + * intervals, and checking if the standard deviation of that set + * of points is below a threshold. If it is... then use the + * average of these 8 points as the estimated value. + */ +static unsigned int get_typical_interval(struct menu_device *data) +{ + int i, divisor; + unsigned int min, max, thresh, avg; + uint64_t sum, variance; + + thresh = INT_MAX; /* Discard outliers above this value */ + +again: + + /* First calculate the average of past intervals */ + min = UINT_MAX; + max = 0; + sum = 0; + divisor = 0; + for (i = 0; i < INTERVALS; i++) { + unsigned int value = data->intervals[i]; + if (value <= thresh) { + sum += value; + divisor++; + if (value > max) + max = value; + + if (value < min) + min = value; + } + } + + if (!max) + return UINT_MAX; + + if (divisor == INTERVALS) + avg = sum >> INTERVAL_SHIFT; + else + avg = div_u64(sum, divisor); + + /* Then try to determine variance */ + variance = 0; + for (i = 0; i < INTERVALS; i++) { + unsigned int value = data->intervals[i]; + if (value <= thresh) { + int64_t diff = (int64_t)value - avg; + variance += diff * diff; + } + } + if (divisor == INTERVALS) + variance >>= INTERVAL_SHIFT; + else + do_div(variance, divisor); + + /* + * The typical interval is obtained when standard deviation is + * small (stddev <= 20 us, variance <= 400 us^2) or standard + * deviation is small compared to the average interval (avg > + * 6*stddev, avg^2 > 36*variance). The average is smaller than + * UINT_MAX aka U32_MAX, so computing its square does not + * overflow a u64. We simply reject this candidate average if + * the standard deviation is greater than 715 s (which is + * rather unlikely). + * + * Use this result only if there is no timer to wake us up sooner. + */ + if (likely(variance <= U64_MAX/36)) { + if ((((u64)avg*avg > variance*36) && (divisor * 4 >= INTERVALS * 3)) + || variance <= 400) { + return avg; + } + } + + /* + * If we have outliers to the upside in our distribution, discard + * those by setting the threshold to exclude these outliers, then + * calculate the average and standard deviation again. Once we get + * down to the bottom 3/4 of our samples, stop excluding samples. + * + * This can deal with workloads that have long pauses interspersed + * with sporadic activity with a bunch of short pauses. + */ + if ((divisor * 4) <= INTERVALS * 3) + return UINT_MAX; + + thresh = max - 1; + goto again; +} + +/** + * menu_select - selects the next idle state to enter + * @drv: cpuidle driver containing state data + * @dev: the CPU + * @stop_tick: indication on whether or not to stop the tick + */ +static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, + bool *stop_tick) +{ + struct menu_device *data = this_cpu_ptr(&menu_devices); + s64 latency_req = cpuidle_governor_latency_req(dev->cpu); + u64 predicted_ns; + u64 interactivity_req; + unsigned int nr_iowaiters; + ktime_t delta, delta_tick; + int i, idx; + + if (data->needs_update) { + menu_update(drv, dev); + data->needs_update = 0; + } + + nr_iowaiters = nr_iowait_cpu(dev->cpu); + + /* Find the shortest expected idle interval. */ + predicted_ns = get_typical_interval(data) * NSEC_PER_USEC; + if (predicted_ns > RESIDENCY_THRESHOLD_NS) { + unsigned int timer_us; + + /* Determine the time till the closest timer. */ + delta = tick_nohz_get_sleep_length(&delta_tick); + if (unlikely(delta < 0)) { + delta = 0; + delta_tick = 0; + } + + data->next_timer_ns = delta; + data->bucket = which_bucket(data->next_timer_ns, nr_iowaiters); + + /* Round up the result for half microseconds. */ + timer_us = div_u64((RESOLUTION * DECAY * NSEC_PER_USEC) / 2 + + data->next_timer_ns * + data->correction_factor[data->bucket], + RESOLUTION * DECAY * NSEC_PER_USEC); + /* Use the lowest expected idle interval to pick the idle state. */ + predicted_ns = min((u64)timer_us * NSEC_PER_USEC, predicted_ns); + } else { + /* + * Because the next timer event is not going to be determined + * in this case, assume that without the tick the closest timer + * will be in distant future and that the closest tick will occur + * after 1/2 of the tick period. + */ + data->next_timer_ns = KTIME_MAX; + delta_tick = TICK_NSEC / 2; + data->bucket = which_bucket(KTIME_MAX, nr_iowaiters); + } + + if (unlikely(drv->state_count <= 1 || latency_req == 0) || + ((data->next_timer_ns < drv->states[1].target_residency_ns || + latency_req < drv->states[1].exit_latency_ns) && + !dev->states_usage[0].disable)) { + /* + * In this case state[0] will be used no matter what, so return + * it right away and keep the tick running if state[0] is a + * polling one. + */ + *stop_tick = !(drv->states[0].flags & CPUIDLE_FLAG_POLLING); + return 0; + } + + if (tick_nohz_tick_stopped()) { + /* + * If the tick is already stopped, the cost of possible short + * idle duration misprediction is much higher, because the CPU + * may be stuck in a shallow idle state for a long time as a + * result of it. In that case say we might mispredict and use + * the known time till the closest timer event for the idle + * state selection. + */ + if (predicted_ns < TICK_NSEC) + predicted_ns = data->next_timer_ns; + } else { + /* + * Use the performance multiplier and the user-configurable + * latency_req to determine the maximum exit latency. + */ + interactivity_req = div64_u64(predicted_ns, + performance_multiplier(nr_iowaiters)); + if (latency_req > interactivity_req) + latency_req = interactivity_req; + } + + /* + * Find the idle state with the lowest power while satisfying + * our constraints. + */ + idx = -1; + for (i = 0; i < drv->state_count; i++) { + struct cpuidle_state *s = &drv->states[i]; + + if (dev->states_usage[i].disable) + continue; + + if (idx == -1) + idx = i; /* first enabled state */ + + if (s->target_residency_ns > predicted_ns) { + /* + * Use a physical idle state, not busy polling, unless + * a timer is going to trigger soon enough. + */ + if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && + s->exit_latency_ns <= latency_req && + s->target_residency_ns <= data->next_timer_ns) { + predicted_ns = s->target_residency_ns; + idx = i; + break; + } + if (predicted_ns < TICK_NSEC) + break; + + if (!tick_nohz_tick_stopped()) { + /* + * If the state selected so far is shallow, + * waking up early won't hurt, so retain the + * tick in that case and let the governor run + * again in the next iteration of the loop. + */ + predicted_ns = drv->states[idx].target_residency_ns; + break; + } + + /* + * If the state selected so far is shallow and this + * state's target residency matches the time till the + * closest timer event, select this one to avoid getting + * stuck in the shallow one for too long. + */ + if (drv->states[idx].target_residency_ns < TICK_NSEC && + s->target_residency_ns <= delta_tick) + idx = i; + + return idx; + } + if (s->exit_latency_ns > latency_req) + break; + + idx = i; + } + + if (idx == -1) + idx = 0; /* No states enabled. Must use 0. */ + + /* + * Don't stop the tick if the selected state is a polling one or if the + * expected idle duration is shorter than the tick period length. + */ + if (((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) || + predicted_ns < TICK_NSEC) && !tick_nohz_tick_stopped()) { + *stop_tick = false; + + if (idx > 0 && drv->states[idx].target_residency_ns > delta_tick) { + /* + * The tick is not going to be stopped and the target + * residency of the state to be returned is not within + * the time until the next timer event including the + * tick, so try to correct that. + */ + for (i = idx - 1; i >= 0; i--) { + if (dev->states_usage[i].disable) + continue; + + idx = i; + if (drv->states[i].target_residency_ns <= delta_tick) + break; + } + } + } + + return idx; +} + +/** + * menu_reflect - records that data structures need update + * @dev: the CPU + * @index: the index of actual entered state + * + * NOTE: it's important to be fast here because this operation will add to + * the overall exit latency. + */ +static void menu_reflect(struct cpuidle_device *dev, int index) +{ + struct menu_device *data = this_cpu_ptr(&menu_devices); + + dev->last_state_idx = index; + data->needs_update = 1; + data->tick_wakeup = tick_nohz_idle_got_tick(); +} + +/** + * menu_update - attempts to guess what happened after entry + * @drv: cpuidle driver containing state data + * @dev: the CPU + */ +static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) +{ + struct menu_device *data = this_cpu_ptr(&menu_devices); + int last_idx = dev->last_state_idx; + struct cpuidle_state *target = &drv->states[last_idx]; + u64 measured_ns; + unsigned int new_factor; + + /* + * Try to figure out how much time passed between entry to low + * power state and occurrence of the wakeup event. + * + * If the entered idle state didn't support residency measurements, + * we use them anyway if they are short, and if long, + * truncate to the whole expected time. + * + * Any measured amount of time will include the exit latency. + * Since we are interested in when the wakeup begun, not when it + * was completed, we must subtract the exit latency. However, if + * the measured amount of time is less than the exit latency, + * assume the state was never reached and the exit latency is 0. + */ + + if (data->tick_wakeup && data->next_timer_ns > TICK_NSEC) { + /* + * The nohz code said that there wouldn't be any events within + * the tick boundary (if the tick was stopped), but the idle + * duration predictor had a differing opinion. Since the CPU + * was woken up by a tick (that wasn't stopped after all), the + * predictor was not quite right, so assume that the CPU could + * have been idle long (but not forever) to help the idle + * duration predictor do a better job next time. + */ + measured_ns = 9 * MAX_INTERESTING / 10; + } else if ((drv->states[last_idx].flags & CPUIDLE_FLAG_POLLING) && + dev->poll_time_limit) { + /* + * The CPU exited the "polling" state due to a time limit, so + * the idle duration prediction leading to the selection of that + * state was inaccurate. If a better prediction had been made, + * the CPU might have been woken up from idle by the next timer. + * Assume that to be the case. + */ + measured_ns = data->next_timer_ns; + } else { + /* measured value */ + measured_ns = dev->last_residency_ns; + + /* Deduct exit latency */ + if (measured_ns > 2 * target->exit_latency_ns) + measured_ns -= target->exit_latency_ns; + else + measured_ns /= 2; + } + + /* Make sure our coefficients do not exceed unity */ + if (measured_ns > data->next_timer_ns) + measured_ns = data->next_timer_ns; + + /* Update our correction ratio */ + new_factor = data->correction_factor[data->bucket]; + new_factor -= new_factor / DECAY; + + if (data->next_timer_ns > 0 && measured_ns < MAX_INTERESTING) + new_factor += div64_u64(RESOLUTION * measured_ns, + data->next_timer_ns); + else + /* + * we were idle so long that we count it as a perfect + * prediction + */ + new_factor += RESOLUTION; + + /* + * We don't want 0 as factor; we always want at least + * a tiny bit of estimated time. Fortunately, due to rounding, + * new_factor will stay nonzero regardless of measured_us values + * and the compiler can eliminate this test as long as DECAY > 1. + */ + if (DECAY == 1 && unlikely(new_factor == 0)) + new_factor = 1; + + data->correction_factor[data->bucket] = new_factor; + + /* update the repeating-pattern data */ + data->intervals[data->interval_ptr++] = ktime_to_us(measured_ns); + if (data->interval_ptr >= INTERVALS) + data->interval_ptr = 0; +} + +/** + * menu_enable_device - scans a CPU's states and does setup + * @drv: cpuidle driver + * @dev: the CPU + */ +static int menu_enable_device(struct cpuidle_driver *drv, + struct cpuidle_device *dev) +{ + struct menu_device *data = &per_cpu(menu_devices, dev->cpu); + int i; + + memset(data, 0, sizeof(struct menu_device)); + + /* + * if the correction factor is 0 (eg first time init or cpu hotplug + * etc), we actually want to start out with a unity factor. + */ + for(i = 0; i < BUCKETS; i++) + data->correction_factor[i] = RESOLUTION * DECAY; + + return 0; +} + +static struct cpuidle_governor menu_governor = { + .name = "menu", + .rating = 20, + .enable = menu_enable_device, + .select = menu_select, + .reflect = menu_reflect, +}; + +/** + * init_menu - initializes the governor + */ +static int __init init_menu(void) +{ + return cpuidle_register_governor(&menu_governor); +} + +postcore_initcall(init_menu); diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c new file mode 100644 index 0000000000..7244f71c59 --- /dev/null +++ b/drivers/cpuidle/governors/teo.c @@ -0,0 +1,695 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Timer events oriented CPU idle governor + * + * TEO governor: + * Copyright (C) 2018 - 2021 Intel Corporation + * Author: Rafael J. Wysocki + * + * Util-awareness mechanism: + * Copyright (C) 2022 Arm Ltd. + * Author: Kajetan Puchalski + */ + +/** + * DOC: teo-description + * + * The idea of this governor is based on the observation that on many systems + * timer events are two or more orders of magnitude more frequent than any + * other interrupts, so they are likely to be the most significant cause of CPU + * wakeups from idle states. Moreover, information about what happened in the + * (relatively recent) past can be used to estimate whether or not the deepest + * idle state with target residency within the (known) time till the closest + * timer event, referred to as the sleep length, is likely to be suitable for + * the upcoming CPU idle period and, if not, then which of the shallower idle + * states to choose instead of it. + * + * Of course, non-timer wakeup sources are more important in some use cases + * which can be covered by taking a few most recent idle time intervals of the + * CPU into account. However, even in that context it is not necessary to + * consider idle duration values greater than the sleep length, because the + * closest timer will ultimately wake up the CPU anyway unless it is woken up + * earlier. + * + * Thus this governor estimates whether or not the prospective idle duration of + * a CPU is likely to be significantly shorter than the sleep length and selects + * an idle state for it accordingly. + * + * The computations carried out by this governor are based on using bins whose + * boundaries are aligned with the target residency parameter values of the CPU + * idle states provided by the %CPUIdle driver in the ascending order. That is, + * the first bin spans from 0 up to, but not including, the target residency of + * the second idle state (idle state 1), the second bin spans from the target + * residency of idle state 1 up to, but not including, the target residency of + * idle state 2, the third bin spans from the target residency of idle state 2 + * up to, but not including, the target residency of idle state 3 and so on. + * The last bin spans from the target residency of the deepest idle state + * supplied by the driver to infinity. + * + * Two metrics called "hits" and "intercepts" are associated with each bin. + * They are updated every time before selecting an idle state for the given CPU + * in accordance with what happened last time. + * + * The "hits" metric reflects the relative frequency of situations in which the + * sleep length and the idle duration measured after CPU wakeup fall into the + * same bin (that is, the CPU appears to wake up "on time" relative to the sleep + * length). In turn, the "intercepts" metric reflects the relative frequency of + * situations in which the measured idle duration is so much shorter than the + * sleep length that the bin it falls into corresponds to an idle state + * shallower than the one whose bin is fallen into by the sleep length (these + * situations are referred to as "intercepts" below). + * + * In addition to the metrics described above, the governor counts recent + * intercepts (that is, intercepts that have occurred during the last + * %NR_RECENT invocations of it for the given CPU) for each bin. + * + * In order to select an idle state for a CPU, the governor takes the following + * steps (modulo the possible latency constraint that must be taken into account + * too): + * + * 1. Find the deepest CPU idle state whose target residency does not exceed + * the current sleep length (the candidate idle state) and compute 3 sums as + * follows: + * + * - The sum of the "hits" and "intercepts" metrics for the candidate state + * and all of the deeper idle states (it represents the cases in which the + * CPU was idle long enough to avoid being intercepted if the sleep length + * had been equal to the current one). + * + * - The sum of the "intercepts" metrics for all of the idle states shallower + * than the candidate one (it represents the cases in which the CPU was not + * idle long enough to avoid being intercepted if the sleep length had been + * equal to the current one). + * + * - The sum of the numbers of recent intercepts for all of the idle states + * shallower than the candidate one. + * + * 2. If the second sum is greater than the first one or the third sum is + * greater than %NR_RECENT / 2, the CPU is likely to wake up early, so look + * for an alternative idle state to select. + * + * - Traverse the idle states shallower than the candidate one in the + * descending order. + * + * - For each of them compute the sum of the "intercepts" metrics and the sum + * of the numbers of recent intercepts over all of the idle states between + * it and the candidate one (including the former and excluding the + * latter). + * + * - If each of these sums that needs to be taken into account (because the + * check related to it has indicated that the CPU is likely to wake up + * early) is greater than a half of the corresponding sum computed in step + * 1 (which means that the target residency of the state in question had + * not exceeded the idle duration in over a half of the relevant cases), + * select the given idle state instead of the candidate one. + * + * 3. By default, select the candidate state. + * + * Util-awareness mechanism: + * + * The idea behind the util-awareness extension is that there are two distinct + * scenarios for the CPU which should result in two different approaches to idle + * state selection - utilized and not utilized. + * + * In this case, 'utilized' means that the average runqueue util of the CPU is + * above a certain threshold. + * + * When the CPU is utilized while going into idle, more likely than not it will + * be woken up to do more work soon and so a shallower idle state should be + * selected to minimise latency and maximise performance. When the CPU is not + * being utilized, the usual metrics-based approach to selecting the deepest + * available idle state should be preferred to take advantage of the power + * saving. + * + * In order to achieve this, the governor uses a utilization threshold. + * The threshold is computed per-CPU as a percentage of the CPU's capacity + * by bit shifting the capacity value. Based on testing, the shift of 6 (~1.56%) + * seems to be getting the best results. + * + * Before selecting the next idle state, the governor compares the current CPU + * util to the precomputed util threshold. If it's below, it defaults to the + * TEO metrics mechanism. If it's above, the closest shallower idle state will + * be selected instead, as long as is not a polling state. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "gov.h" + +/* + * The number of bits to shift the CPU's capacity by in order to determine + * the utilized threshold. + * + * 6 was chosen based on testing as the number that achieved the best balance + * of power and performance on average. + * + * The resulting threshold is high enough to not be triggered by background + * noise and low enough to react quickly when activity starts to ramp up. + */ +#define UTIL_THRESHOLD_SHIFT 6 + +/* + * The PULSE value is added to metrics when they grow and the DECAY_SHIFT value + * is used for decreasing metrics on a regular basis. + */ +#define PULSE 1024 +#define DECAY_SHIFT 3 + +/* + * Number of the most recent idle duration values to take into consideration for + * the detection of recent early wakeup patterns. + */ +#define NR_RECENT 9 + +/** + * struct teo_bin - Metrics used by the TEO cpuidle governor. + * @intercepts: The "intercepts" metric. + * @hits: The "hits" metric. + * @recent: The number of recent "intercepts". + */ +struct teo_bin { + unsigned int intercepts; + unsigned int hits; + unsigned int recent; +}; + +/** + * struct teo_cpu - CPU data used by the TEO cpuidle governor. + * @time_span_ns: Time between idle state selection and post-wakeup update. + * @sleep_length_ns: Time till the closest timer event (at the selection time). + * @state_bins: Idle state data bins for this CPU. + * @total: Grand total of the "intercepts" and "hits" metrics for all bins. + * @next_recent_idx: Index of the next @recent_idx entry to update. + * @recent_idx: Indices of bins corresponding to recent "intercepts". + * @tick_hits: Number of "hits" after TICK_NSEC. + * @util_threshold: Threshold above which the CPU is considered utilized + */ +struct teo_cpu { + s64 time_span_ns; + s64 sleep_length_ns; + struct teo_bin state_bins[CPUIDLE_STATE_MAX]; + unsigned int total; + int next_recent_idx; + int recent_idx[NR_RECENT]; + unsigned int tick_hits; + unsigned long util_threshold; +}; + +static DEFINE_PER_CPU(struct teo_cpu, teo_cpus); + +/** + * teo_cpu_is_utilized - Check if the CPU's util is above the threshold + * @cpu: Target CPU + * @cpu_data: Governor CPU data for the target CPU + */ +#ifdef CONFIG_SMP +static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data) +{ + return sched_cpu_util(cpu) > cpu_data->util_threshold; +} +#else +static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data) +{ + return false; +} +#endif + +/** + * teo_update - Update CPU metrics after wakeup. + * @drv: cpuidle driver containing state data. + * @dev: Target CPU. + */ +static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) +{ + struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + int i, idx_timer = 0, idx_duration = 0; + s64 target_residency_ns; + u64 measured_ns; + + if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns) { + /* + * One of the safety nets has triggered or the wakeup was close + * enough to the closest timer event expected at the idle state + * selection time to be discarded. + */ + measured_ns = U64_MAX; + } else { + u64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns; + + /* + * The computations below are to determine whether or not the + * (saved) time till the next timer event and the measured idle + * duration fall into the same "bin", so use last_residency_ns + * for that instead of time_span_ns which includes the cpuidle + * overhead. + */ + measured_ns = dev->last_residency_ns; + /* + * The delay between the wakeup and the first instruction + * executed by the CPU is not likely to be worst-case every + * time, so take 1/2 of the exit latency as a very rough + * approximation of the average of it. + */ + if (measured_ns >= lat_ns) + measured_ns -= lat_ns / 2; + else + measured_ns /= 2; + } + + cpu_data->total = 0; + + /* + * Decay the "hits" and "intercepts" metrics for all of the bins and + * find the bins that the sleep length and the measured idle duration + * fall into. + */ + for (i = 0; i < drv->state_count; i++) { + struct teo_bin *bin = &cpu_data->state_bins[i]; + + bin->hits -= bin->hits >> DECAY_SHIFT; + bin->intercepts -= bin->intercepts >> DECAY_SHIFT; + + cpu_data->total += bin->hits + bin->intercepts; + + target_residency_ns = drv->states[i].target_residency_ns; + + if (target_residency_ns <= cpu_data->sleep_length_ns) { + idx_timer = i; + if (target_residency_ns <= measured_ns) + idx_duration = i; + } + } + + i = cpu_data->next_recent_idx++; + if (cpu_data->next_recent_idx >= NR_RECENT) + cpu_data->next_recent_idx = 0; + + if (cpu_data->recent_idx[i] >= 0) + cpu_data->state_bins[cpu_data->recent_idx[i]].recent--; + + /* + * If the deepest state's target residency is below the tick length, + * make a record of it to help teo_select() decide whether or not + * to stop the tick. This effectively adds an extra hits-only bin + * beyond the last state-related one. + */ + if (target_residency_ns < TICK_NSEC) { + cpu_data->tick_hits -= cpu_data->tick_hits >> DECAY_SHIFT; + + cpu_data->total += cpu_data->tick_hits; + + if (TICK_NSEC <= cpu_data->sleep_length_ns) { + idx_timer = drv->state_count; + if (TICK_NSEC <= measured_ns) { + cpu_data->tick_hits += PULSE; + goto end; + } + } + } + + /* + * If the measured idle duration falls into the same bin as the sleep + * length, this is a "hit", so update the "hits" metric for that bin. + * Otherwise, update the "intercepts" metric for the bin fallen into by + * the measured idle duration. + */ + if (idx_timer == idx_duration) { + cpu_data->state_bins[idx_timer].hits += PULSE; + cpu_data->recent_idx[i] = -1; + } else { + cpu_data->state_bins[idx_duration].intercepts += PULSE; + cpu_data->state_bins[idx_duration].recent++; + cpu_data->recent_idx[i] = idx_duration; + } + +end: + cpu_data->total += PULSE; +} + +static bool teo_state_ok(int i, struct cpuidle_driver *drv) +{ + return !tick_nohz_tick_stopped() || + drv->states[i].target_residency_ns >= TICK_NSEC; +} + +/** + * teo_find_shallower_state - Find shallower idle state matching given duration. + * @drv: cpuidle driver containing state data. + * @dev: Target CPU. + * @state_idx: Index of the capping idle state. + * @duration_ns: Idle duration value to match. + * @no_poll: Don't consider polling states. + */ +static int teo_find_shallower_state(struct cpuidle_driver *drv, + struct cpuidle_device *dev, int state_idx, + s64 duration_ns, bool no_poll) +{ + int i; + + for (i = state_idx - 1; i >= 0; i--) { + if (dev->states_usage[i].disable || + (no_poll && drv->states[i].flags & CPUIDLE_FLAG_POLLING)) + continue; + + state_idx = i; + if (drv->states[i].target_residency_ns <= duration_ns) + break; + } + return state_idx; +} + +/** + * teo_select - Selects the next idle state to enter. + * @drv: cpuidle driver containing state data. + * @dev: Target CPU. + * @stop_tick: Indication on whether or not to stop the scheduler tick. + */ +static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, + bool *stop_tick) +{ + struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + s64 latency_req = cpuidle_governor_latency_req(dev->cpu); + ktime_t delta_tick = TICK_NSEC / 2; + unsigned int tick_intercept_sum = 0; + unsigned int idx_intercept_sum = 0; + unsigned int intercept_sum = 0; + unsigned int idx_recent_sum = 0; + unsigned int recent_sum = 0; + unsigned int idx_hit_sum = 0; + unsigned int hit_sum = 0; + int constraint_idx = 0; + int idx0 = 0, idx = -1; + bool alt_intercepts, alt_recent; + bool cpu_utilized; + s64 duration_ns; + int i; + + if (dev->last_state_idx >= 0) { + teo_update(drv, dev); + dev->last_state_idx = -1; + } + + cpu_data->time_span_ns = local_clock(); + /* + * Set the expected sleep length to infinity in case of an early + * return. + */ + cpu_data->sleep_length_ns = KTIME_MAX; + + /* Check if there is any choice in the first place. */ + if (drv->state_count < 2) { + idx = 0; + goto out_tick; + } + + if (!dev->states_usage[0].disable) + idx = 0; + + cpu_utilized = teo_cpu_is_utilized(dev->cpu, cpu_data); + /* + * If the CPU is being utilized over the threshold and there are only 2 + * states to choose from, the metrics need not be considered, so choose + * the shallowest non-polling state and exit. + */ + if (drv->state_count < 3 && cpu_utilized) { + /* + * If state 0 is enabled and it is not a polling one, select it + * right away unless the scheduler tick has been stopped, in + * which case care needs to be taken to leave the CPU in a deep + * enough state in case it is not woken up any time soon after + * all. If state 1 is disabled, though, state 0 must be used + * anyway. + */ + if ((!idx && !(drv->states[0].flags & CPUIDLE_FLAG_POLLING) && + teo_state_ok(0, drv)) || dev->states_usage[1].disable) { + idx = 0; + goto out_tick; + } + /* Assume that state 1 is not a polling one and use it. */ + idx = 1; + duration_ns = drv->states[1].target_residency_ns; + goto end; + } + + /* Compute the sums of metrics for early wakeup pattern detection. */ + for (i = 1; i < drv->state_count; i++) { + struct teo_bin *prev_bin = &cpu_data->state_bins[i-1]; + struct cpuidle_state *s = &drv->states[i]; + + /* + * Update the sums of idle state mertics for all of the states + * shallower than the current one. + */ + intercept_sum += prev_bin->intercepts; + hit_sum += prev_bin->hits; + recent_sum += prev_bin->recent; + + if (dev->states_usage[i].disable) + continue; + + if (idx < 0) + idx0 = i; /* first enabled state */ + + idx = i; + + if (s->exit_latency_ns <= latency_req) + constraint_idx = i; + + /* Save the sums for the current state. */ + idx_intercept_sum = intercept_sum; + idx_hit_sum = hit_sum; + idx_recent_sum = recent_sum; + } + + /* Avoid unnecessary overhead. */ + if (idx < 0) { + idx = 0; /* No states enabled, must use 0. */ + goto out_tick; + } + + if (idx == idx0) { + /* + * Only one idle state is enabled, so use it, but do not + * allow the tick to be stopped it is shallow enough. + */ + duration_ns = drv->states[idx].target_residency_ns; + goto end; + } + + tick_intercept_sum = intercept_sum + + cpu_data->state_bins[drv->state_count-1].intercepts; + + /* + * If the sum of the intercepts metric for all of the idle states + * shallower than the current candidate one (idx) is greater than the + * sum of the intercepts and hits metrics for the candidate state and + * all of the deeper states, or the sum of the numbers of recent + * intercepts over all of the states shallower than the candidate one + * is greater than a half of the number of recent events taken into + * account, a shallower idle state is likely to be a better choice. + */ + alt_intercepts = 2 * idx_intercept_sum > cpu_data->total - idx_hit_sum; + alt_recent = idx_recent_sum > NR_RECENT / 2; + if (alt_recent || alt_intercepts) { + int first_suitable_idx = idx; + + /* + * Look for the deepest idle state whose target residency had + * not exceeded the idle duration in over a half of the relevant + * cases (both with respect to intercepts overall and with + * respect to the recent intercepts only) in the past. + * + * Take the possible duration limitation present if the tick + * has been stopped already into account. + */ + intercept_sum = 0; + recent_sum = 0; + + for (i = idx - 1; i >= 0; i--) { + struct teo_bin *bin = &cpu_data->state_bins[i]; + + intercept_sum += bin->intercepts; + recent_sum += bin->recent; + + if ((!alt_recent || 2 * recent_sum > idx_recent_sum) && + (!alt_intercepts || + 2 * intercept_sum > idx_intercept_sum)) { + /* + * Use the current state unless it is too + * shallow or disabled, in which case take the + * first enabled state that is deep enough. + */ + if (teo_state_ok(i, drv) && + !dev->states_usage[i].disable) + idx = i; + else + idx = first_suitable_idx; + + break; + } + + if (dev->states_usage[i].disable) + continue; + + if (!teo_state_ok(i, drv)) { + /* + * The current state is too shallow, but if an + * alternative candidate state has been found, + * it may still turn out to be a better choice. + */ + if (first_suitable_idx != idx) + continue; + + break; + } + + first_suitable_idx = i; + } + } + + /* + * If there is a latency constraint, it may be necessary to select an + * idle state shallower than the current candidate one. + */ + if (idx > constraint_idx) + idx = constraint_idx; + + /* + * If the CPU is being utilized over the threshold, choose a shallower + * non-polling state to improve latency, unless the scheduler tick has + * been stopped already and the shallower state's target residency is + * not sufficiently large. + */ + if (cpu_utilized) { + i = teo_find_shallower_state(drv, dev, idx, KTIME_MAX, true); + if (teo_state_ok(i, drv)) + idx = i; + } + + /* + * Skip the timers check if state 0 is the current candidate one, + * because an immediate non-timer wakeup is expected in that case. + */ + if (!idx) + goto out_tick; + + /* + * If state 0 is a polling one, check if the target residency of + * the current candidate state is low enough and skip the timers + * check in that case too. + */ + if ((drv->states[0].flags & CPUIDLE_FLAG_POLLING) && + drv->states[idx].target_residency_ns < RESIDENCY_THRESHOLD_NS) + goto out_tick; + + duration_ns = tick_nohz_get_sleep_length(&delta_tick); + cpu_data->sleep_length_ns = duration_ns; + + /* + * If the closest expected timer is before the terget residency of the + * candidate state, a shallower one needs to be found. + */ + if (drv->states[idx].target_residency_ns > duration_ns) { + i = teo_find_shallower_state(drv, dev, idx, duration_ns, false); + if (teo_state_ok(i, drv)) + idx = i; + } + + /* + * If the selected state's target residency is below the tick length + * and intercepts occurring before the tick length are the majority of + * total wakeup events, do not stop the tick. + */ + if (drv->states[idx].target_residency_ns < TICK_NSEC && + tick_intercept_sum > cpu_data->total / 2 + cpu_data->total / 8) + duration_ns = TICK_NSEC / 2; + +end: + /* + * Allow the tick to be stopped unless the selected state is a polling + * one or the expected idle duration is shorter than the tick period + * length. + */ + if ((!(drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && + duration_ns >= TICK_NSEC) || tick_nohz_tick_stopped()) + return idx; + + /* + * The tick is not going to be stopped, so if the target residency of + * the state to be returned is not within the time till the closest + * timer including the tick, try to correct that. + */ + if (idx > idx0 && + drv->states[idx].target_residency_ns > delta_tick) + idx = teo_find_shallower_state(drv, dev, idx, delta_tick, false); + +out_tick: + *stop_tick = false; + return idx; +} + +/** + * teo_reflect - Note that governor data for the CPU need to be updated. + * @dev: Target CPU. + * @state: Entered state. + */ +static void teo_reflect(struct cpuidle_device *dev, int state) +{ + struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + + dev->last_state_idx = state; + /* + * If the wakeup was not "natural", but triggered by one of the safety + * nets, assume that the CPU might have been idle for the entire sleep + * length time. + */ + if (dev->poll_time_limit || + (tick_nohz_idle_got_tick() && cpu_data->sleep_length_ns > TICK_NSEC)) { + dev->poll_time_limit = false; + cpu_data->time_span_ns = cpu_data->sleep_length_ns; + } else { + cpu_data->time_span_ns = local_clock() - cpu_data->time_span_ns; + } +} + +/** + * teo_enable_device - Initialize the governor's data for the target CPU. + * @drv: cpuidle driver (not used). + * @dev: Target CPU. + */ +static int teo_enable_device(struct cpuidle_driver *drv, + struct cpuidle_device *dev) +{ + struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + unsigned long max_capacity = arch_scale_cpu_capacity(dev->cpu); + int i; + + memset(cpu_data, 0, sizeof(*cpu_data)); + cpu_data->util_threshold = max_capacity >> UTIL_THRESHOLD_SHIFT; + + for (i = 0; i < NR_RECENT; i++) + cpu_data->recent_idx[i] = -1; + + return 0; +} + +static struct cpuidle_governor teo_governor = { + .name = "teo", + .rating = 19, + .enable = teo_enable_device, + .select = teo_select, + .reflect = teo_reflect, +}; + +static int __init teo_governor_init(void) +{ + return cpuidle_register_governor(&teo_governor); +} + +postcore_initcall(teo_governor_init); -- cgit v1.2.3