summaryrefslogtreecommitdiffstats
path: root/drivers/cpuidle/governors
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/cpuidle/governors')
-rw-r--r--drivers/cpuidle/governors/Makefile9
-rw-r--r--drivers/cpuidle/governors/gov.h14
-rw-r--r--drivers/cpuidle/governors/haltpoll.c152
-rw-r--r--drivers/cpuidle/governors/ladder.c197
-rw-r--r--drivers/cpuidle/governors/menu.c590
-rw-r--r--drivers/cpuidle/governors/teo.c695
6 files changed, 1657 insertions, 0 deletions
diff --git a/drivers/cpuidle/governors/Makefile b/drivers/cpuidle/governors/Makefile
new file mode 100644
index 0000000000..63abb5393a
--- /dev/null
+++ b/drivers/cpuidle/governors/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for cpuidle governors.
+#
+
+obj-$(CONFIG_CPU_IDLE_GOV_LADDER) += ladder.o
+obj-$(CONFIG_CPU_IDLE_GOV_MENU) += menu.o
+obj-$(CONFIG_CPU_IDLE_GOV_TEO) += teo.o
+obj-$(CONFIG_CPU_IDLE_GOV_HALTPOLL) += haltpoll.o
diff --git a/drivers/cpuidle/governors/gov.h b/drivers/cpuidle/governors/gov.h
new file mode 100644
index 0000000000..99e067d966
--- /dev/null
+++ b/drivers/cpuidle/governors/gov.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/* Common definitions for cpuidle governors. */
+
+#ifndef __CPUIDLE_GOVERNOR_H
+#define __CPUIDLE_GOVERNOR_H
+
+/*
+ * Idle state target residency threshold used for deciding whether or not to
+ * check the time till the closest expected timer event.
+ */
+#define RESIDENCY_THRESHOLD_NS (15 * NSEC_PER_USEC)
+
+#endif /* __CPUIDLE_GOVERNOR_H */
diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c
new file mode 100644
index 0000000000..1dff3a5291
--- /dev/null
+++ b/drivers/cpuidle/governors/haltpoll.c
@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * haltpoll.c - haltpoll idle governor
+ *
+ * Copyright 2019 Red Hat, Inc. and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * Authors: Marcelo Tosatti <mtosatti@redhat.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/cpuidle.h>
+#include <linux/time.h>
+#include <linux/ktime.h>
+#include <linux/hrtimer.h>
+#include <linux/tick.h>
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/kvm_para.h>
+#include <trace/events/power.h>
+
+static unsigned int guest_halt_poll_ns __read_mostly = 200000;
+module_param(guest_halt_poll_ns, uint, 0644);
+
+/* division factor to shrink halt_poll_ns */
+static unsigned int guest_halt_poll_shrink __read_mostly = 2;
+module_param(guest_halt_poll_shrink, uint, 0644);
+
+/* multiplication factor to grow per-cpu poll_limit_ns */
+static unsigned int guest_halt_poll_grow __read_mostly = 2;
+module_param(guest_halt_poll_grow, uint, 0644);
+
+/* value in us to start growing per-cpu halt_poll_ns */
+static unsigned int guest_halt_poll_grow_start __read_mostly = 50000;
+module_param(guest_halt_poll_grow_start, uint, 0644);
+
+/* allow shrinking guest halt poll */
+static bool guest_halt_poll_allow_shrink __read_mostly = true;
+module_param(guest_halt_poll_allow_shrink, bool, 0644);
+
+/**
+ * haltpoll_select - selects the next idle state to enter
+ * @drv: cpuidle driver containing state data
+ * @dev: the CPU
+ * @stop_tick: indication on whether or not to stop the tick
+ */
+static int haltpoll_select(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev,
+ bool *stop_tick)
+{
+ s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
+
+ if (!drv->state_count || latency_req == 0) {
+ *stop_tick = false;
+ return 0;
+ }
+
+ if (dev->poll_limit_ns == 0)
+ return 1;
+
+ /* Last state was poll? */
+ if (dev->last_state_idx == 0) {
+ /* Halt if no event occurred on poll window */
+ if (dev->poll_time_limit == true)
+ return 1;
+
+ *stop_tick = false;
+ /* Otherwise, poll again */
+ return 0;
+ }
+
+ *stop_tick = false;
+ /* Last state was halt: poll */
+ return 0;
+}
+
+static void adjust_poll_limit(struct cpuidle_device *dev, u64 block_ns)
+{
+ unsigned int val;
+
+ /* Grow cpu_halt_poll_us if
+ * cpu_halt_poll_us < block_ns < guest_halt_poll_us
+ */
+ if (block_ns > dev->poll_limit_ns && block_ns <= guest_halt_poll_ns) {
+ val = dev->poll_limit_ns * guest_halt_poll_grow;
+
+ if (val < guest_halt_poll_grow_start)
+ val = guest_halt_poll_grow_start;
+ if (val > guest_halt_poll_ns)
+ val = guest_halt_poll_ns;
+
+ trace_guest_halt_poll_ns_grow(val, dev->poll_limit_ns);
+ dev->poll_limit_ns = val;
+ } else if (block_ns > guest_halt_poll_ns &&
+ guest_halt_poll_allow_shrink) {
+ unsigned int shrink = guest_halt_poll_shrink;
+
+ val = dev->poll_limit_ns;
+ if (shrink == 0)
+ val = 0;
+ else
+ val /= shrink;
+ trace_guest_halt_poll_ns_shrink(val, dev->poll_limit_ns);
+ dev->poll_limit_ns = val;
+ }
+}
+
+/**
+ * haltpoll_reflect - update variables and update poll time
+ * @dev: the CPU
+ * @index: the index of actual entered state
+ */
+static void haltpoll_reflect(struct cpuidle_device *dev, int index)
+{
+ dev->last_state_idx = index;
+
+ if (index != 0)
+ adjust_poll_limit(dev, dev->last_residency_ns);
+}
+
+/**
+ * haltpoll_enable_device - scans a CPU's states and does setup
+ * @drv: cpuidle driver
+ * @dev: the CPU
+ */
+static int haltpoll_enable_device(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev)
+{
+ dev->poll_limit_ns = 0;
+
+ return 0;
+}
+
+static struct cpuidle_governor haltpoll_governor = {
+ .name = "haltpoll",
+ .rating = 9,
+ .enable = haltpoll_enable_device,
+ .select = haltpoll_select,
+ .reflect = haltpoll_reflect,
+};
+
+static int __init init_haltpoll(void)
+{
+ if (kvm_para_available())
+ return cpuidle_register_governor(&haltpoll_governor);
+
+ return 0;
+}
+
+postcore_initcall(init_haltpoll);
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
new file mode 100644
index 0000000000..8e9058c4ea
--- /dev/null
+++ b/drivers/cpuidle/governors/ladder.c
@@ -0,0 +1,197 @@
+/*
+ * ladder.c - the residency ladder algorithm
+ *
+ * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
+ * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
+ * Copyright (C) 2004, 2005 Dominik Brodowski <linux@brodo.de>
+ *
+ * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
+ * Shaohua Li <shaohua.li@intel.com>
+ * Adam Belay <abelay@novell.com>
+ *
+ * This code is licenced under the GPL.
+ */
+
+#include <linux/kernel.h>
+#include <linux/cpuidle.h>
+#include <linux/jiffies.h>
+#include <linux/tick.h>
+
+#include <asm/io.h>
+#include <linux/uaccess.h>
+
+#define PROMOTION_COUNT 4
+#define DEMOTION_COUNT 1
+
+struct ladder_device_state {
+ struct {
+ u32 promotion_count;
+ u32 demotion_count;
+ u64 promotion_time_ns;
+ u64 demotion_time_ns;
+ } threshold;
+ struct {
+ int promotion_count;
+ int demotion_count;
+ } stats;
+};
+
+struct ladder_device {
+ struct ladder_device_state states[CPUIDLE_STATE_MAX];
+};
+
+static DEFINE_PER_CPU(struct ladder_device, ladder_devices);
+
+/**
+ * ladder_do_selection - prepares private data for a state change
+ * @ldev: the ladder device
+ * @old_idx: the current state index
+ * @new_idx: the new target state index
+ */
+static inline void ladder_do_selection(struct cpuidle_device *dev,
+ struct ladder_device *ldev,
+ int old_idx, int new_idx)
+{
+ ldev->states[old_idx].stats.promotion_count = 0;
+ ldev->states[old_idx].stats.demotion_count = 0;
+ dev->last_state_idx = new_idx;
+}
+
+/**
+ * ladder_select_state - selects the next state to enter
+ * @drv: cpuidle driver
+ * @dev: the CPU
+ * @dummy: not used
+ */
+static int ladder_select_state(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev, bool *dummy)
+{
+ struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
+ struct ladder_device_state *last_state;
+ int last_idx = dev->last_state_idx;
+ int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
+ s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
+ s64 last_residency;
+
+ /* Special case when user has set very strict latency requirement */
+ if (unlikely(latency_req == 0)) {
+ ladder_do_selection(dev, ldev, last_idx, 0);
+ return 0;
+ }
+
+ last_state = &ldev->states[last_idx];
+
+ last_residency = dev->last_residency_ns - drv->states[last_idx].exit_latency_ns;
+
+ /* consider promotion */
+ if (last_idx < drv->state_count - 1 &&
+ !dev->states_usage[last_idx + 1].disable &&
+ last_residency > last_state->threshold.promotion_time_ns &&
+ drv->states[last_idx + 1].exit_latency_ns <= latency_req) {
+ last_state->stats.promotion_count++;
+ last_state->stats.demotion_count = 0;
+ if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
+ ladder_do_selection(dev, ldev, last_idx, last_idx + 1);
+ return last_idx + 1;
+ }
+ }
+
+ /* consider demotion */
+ if (last_idx > first_idx &&
+ (dev->states_usage[last_idx].disable ||
+ drv->states[last_idx].exit_latency_ns > latency_req)) {
+ int i;
+
+ for (i = last_idx - 1; i > first_idx; i--) {
+ if (drv->states[i].exit_latency_ns <= latency_req)
+ break;
+ }
+ ladder_do_selection(dev, ldev, last_idx, i);
+ return i;
+ }
+
+ if (last_idx > first_idx &&
+ last_residency < last_state->threshold.demotion_time_ns) {
+ last_state->stats.demotion_count++;
+ last_state->stats.promotion_count = 0;
+ if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
+ ladder_do_selection(dev, ldev, last_idx, last_idx - 1);
+ return last_idx - 1;
+ }
+ }
+
+ /* otherwise remain at the current state */
+ return last_idx;
+}
+
+/**
+ * ladder_enable_device - setup for the governor
+ * @drv: cpuidle driver
+ * @dev: the CPU
+ */
+static int ladder_enable_device(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev)
+{
+ int i;
+ int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
+ struct ladder_device *ldev = &per_cpu(ladder_devices, dev->cpu);
+ struct ladder_device_state *lstate;
+ struct cpuidle_state *state;
+
+ dev->last_state_idx = first_idx;
+
+ for (i = first_idx; i < drv->state_count; i++) {
+ state = &drv->states[i];
+ lstate = &ldev->states[i];
+
+ lstate->stats.promotion_count = 0;
+ lstate->stats.demotion_count = 0;
+
+ lstate->threshold.promotion_count = PROMOTION_COUNT;
+ lstate->threshold.demotion_count = DEMOTION_COUNT;
+
+ if (i < drv->state_count - 1)
+ lstate->threshold.promotion_time_ns = state->exit_latency_ns;
+ if (i > first_idx)
+ lstate->threshold.demotion_time_ns = state->exit_latency_ns;
+ }
+
+ return 0;
+}
+
+/**
+ * ladder_reflect - update the correct last_state_idx
+ * @dev: the CPU
+ * @index: the index of actual state entered
+ */
+static void ladder_reflect(struct cpuidle_device *dev, int index)
+{
+ if (index > 0)
+ dev->last_state_idx = index;
+}
+
+static struct cpuidle_governor ladder_governor = {
+ .name = "ladder",
+ .rating = 10,
+ .enable = ladder_enable_device,
+ .select = ladder_select_state,
+ .reflect = ladder_reflect,
+};
+
+/**
+ * init_ladder - initializes the governor
+ */
+static int __init init_ladder(void)
+{
+ /*
+ * When NO_HZ is disabled, or when booting with nohz=off, the ladder
+ * governor is better so give it a higher rating than the menu
+ * governor.
+ */
+ if (!tick_nohz_enabled)
+ ladder_governor.rating = 25;
+
+ return cpuidle_register_governor(&ladder_governor);
+}
+
+postcore_initcall(init_ladder);
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
new file mode 100644
index 0000000000..b96e3da0fe
--- /dev/null
+++ b/drivers/cpuidle/governors/menu.c
@@ -0,0 +1,590 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * menu.c - the menu idle governor
+ *
+ * Copyright (C) 2006-2007 Adam Belay <abelay@novell.com>
+ * Copyright (C) 2009 Intel Corporation
+ * Author:
+ * Arjan van de Ven <arjan@linux.intel.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/cpuidle.h>
+#include <linux/time.h>
+#include <linux/ktime.h>
+#include <linux/hrtimer.h>
+#include <linux/tick.h>
+#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/stat.h>
+#include <linux/math64.h>
+
+#include "gov.h"
+
+#define BUCKETS 12
+#define INTERVAL_SHIFT 3
+#define INTERVALS (1UL << INTERVAL_SHIFT)
+#define RESOLUTION 1024
+#define DECAY 8
+#define MAX_INTERESTING (50000 * NSEC_PER_USEC)
+
+/*
+ * Concepts and ideas behind the menu governor
+ *
+ * For the menu governor, there are 3 decision factors for picking a C
+ * state:
+ * 1) Energy break even point
+ * 2) Performance impact
+ * 3) Latency tolerance (from pmqos infrastructure)
+ * These three factors are treated independently.
+ *
+ * Energy break even point
+ * -----------------------
+ * C state entry and exit have an energy cost, and a certain amount of time in
+ * the C state is required to actually break even on this cost. CPUIDLE
+ * provides us this duration in the "target_residency" field. So all that we
+ * need is a good prediction of how long we'll be idle. Like the traditional
+ * menu governor, we start with the actual known "next timer event" time.
+ *
+ * Since there are other source of wakeups (interrupts for example) than
+ * the next timer event, this estimation is rather optimistic. To get a
+ * more realistic estimate, a correction factor is applied to the estimate,
+ * that is based on historic behavior. For example, if in the past the actual
+ * duration always was 50% of the next timer tick, the correction factor will
+ * be 0.5.
+ *
+ * menu uses a running average for this correction factor, however it uses a
+ * set of factors, not just a single factor. This stems from the realization
+ * that the ratio is dependent on the order of magnitude of the expected
+ * duration; if we expect 500 milliseconds of idle time the likelihood of
+ * getting an interrupt very early is much higher than if we expect 50 micro
+ * seconds of idle time. A second independent factor that has big impact on
+ * the actual factor is if there is (disk) IO outstanding or not.
+ * (as a special twist, we consider every sleep longer than 50 milliseconds
+ * as perfect; there are no power gains for sleeping longer than this)
+ *
+ * For these two reasons we keep an array of 12 independent factors, that gets
+ * indexed based on the magnitude of the expected duration as well as the
+ * "is IO outstanding" property.
+ *
+ * Repeatable-interval-detector
+ * ----------------------------
+ * There are some cases where "next timer" is a completely unusable predictor:
+ * Those cases where the interval is fixed, for example due to hardware
+ * interrupt mitigation, but also due to fixed transfer rate devices such as
+ * mice.
+ * For this, we use a different predictor: We track the duration of the last 8
+ * intervals and if the stand deviation of these 8 intervals is below a
+ * threshold value, we use the average of these intervals as prediction.
+ *
+ * Limiting Performance Impact
+ * ---------------------------
+ * C states, especially those with large exit latencies, can have a real
+ * noticeable impact on workloads, which is not acceptable for most sysadmins,
+ * and in addition, less performance has a power price of its own.
+ *
+ * As a general rule of thumb, menu assumes that the following heuristic
+ * holds:
+ * The busier the system, the less impact of C states is acceptable
+ *
+ * This rule-of-thumb is implemented using a performance-multiplier:
+ * If the exit latency times the performance multiplier is longer than
+ * the predicted duration, the C state is not considered a candidate
+ * for selection due to a too high performance impact. So the higher
+ * this multiplier is, the longer we need to be idle to pick a deep C
+ * state, and thus the less likely a busy CPU will hit such a deep
+ * C state.
+ *
+ * Two factors are used in determing this multiplier:
+ * a value of 10 is added for each point of "per cpu load average" we have.
+ * a value of 5 points is added for each process that is waiting for
+ * IO on this CPU.
+ * (these values are experimentally determined)
+ *
+ * The load average factor gives a longer term (few seconds) input to the
+ * decision, while the iowait value gives a cpu local instantanious input.
+ * The iowait factor may look low, but realize that this is also already
+ * represented in the system load average.
+ *
+ */
+
+struct menu_device {
+ int needs_update;
+ int tick_wakeup;
+
+ u64 next_timer_ns;
+ unsigned int bucket;
+ unsigned int correction_factor[BUCKETS];
+ unsigned int intervals[INTERVALS];
+ int interval_ptr;
+};
+
+static inline int which_bucket(u64 duration_ns, unsigned int nr_iowaiters)
+{
+ int bucket = 0;
+
+ /*
+ * We keep two groups of stats; one with no
+ * IO pending, one without.
+ * This allows us to calculate
+ * E(duration)|iowait
+ */
+ if (nr_iowaiters)
+ bucket = BUCKETS/2;
+
+ if (duration_ns < 10ULL * NSEC_PER_USEC)
+ return bucket;
+ if (duration_ns < 100ULL * NSEC_PER_USEC)
+ return bucket + 1;
+ if (duration_ns < 1000ULL * NSEC_PER_USEC)
+ return bucket + 2;
+ if (duration_ns < 10000ULL * NSEC_PER_USEC)
+ return bucket + 3;
+ if (duration_ns < 100000ULL * NSEC_PER_USEC)
+ return bucket + 4;
+ return bucket + 5;
+}
+
+/*
+ * Return a multiplier for the exit latency that is intended
+ * to take performance requirements into account.
+ * The more performance critical we estimate the system
+ * to be, the higher this multiplier, and thus the higher
+ * the barrier to go to an expensive C state.
+ */
+static inline int performance_multiplier(unsigned int nr_iowaiters)
+{
+ /* for IO wait tasks (per cpu!) we add 10x each */
+ return 1 + 10 * nr_iowaiters;
+}
+
+static DEFINE_PER_CPU(struct menu_device, menu_devices);
+
+static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev);
+
+/*
+ * Try detecting repeating patterns by keeping track of the last 8
+ * intervals, and checking if the standard deviation of that set
+ * of points is below a threshold. If it is... then use the
+ * average of these 8 points as the estimated value.
+ */
+static unsigned int get_typical_interval(struct menu_device *data)
+{
+ int i, divisor;
+ unsigned int min, max, thresh, avg;
+ uint64_t sum, variance;
+
+ thresh = INT_MAX; /* Discard outliers above this value */
+
+again:
+
+ /* First calculate the average of past intervals */
+ min = UINT_MAX;
+ max = 0;
+ sum = 0;
+ divisor = 0;
+ for (i = 0; i < INTERVALS; i++) {
+ unsigned int value = data->intervals[i];
+ if (value <= thresh) {
+ sum += value;
+ divisor++;
+ if (value > max)
+ max = value;
+
+ if (value < min)
+ min = value;
+ }
+ }
+
+ if (!max)
+ return UINT_MAX;
+
+ if (divisor == INTERVALS)
+ avg = sum >> INTERVAL_SHIFT;
+ else
+ avg = div_u64(sum, divisor);
+
+ /* Then try to determine variance */
+ variance = 0;
+ for (i = 0; i < INTERVALS; i++) {
+ unsigned int value = data->intervals[i];
+ if (value <= thresh) {
+ int64_t diff = (int64_t)value - avg;
+ variance += diff * diff;
+ }
+ }
+ if (divisor == INTERVALS)
+ variance >>= INTERVAL_SHIFT;
+ else
+ do_div(variance, divisor);
+
+ /*
+ * The typical interval is obtained when standard deviation is
+ * small (stddev <= 20 us, variance <= 400 us^2) or standard
+ * deviation is small compared to the average interval (avg >
+ * 6*stddev, avg^2 > 36*variance). The average is smaller than
+ * UINT_MAX aka U32_MAX, so computing its square does not
+ * overflow a u64. We simply reject this candidate average if
+ * the standard deviation is greater than 715 s (which is
+ * rather unlikely).
+ *
+ * Use this result only if there is no timer to wake us up sooner.
+ */
+ if (likely(variance <= U64_MAX/36)) {
+ if ((((u64)avg*avg > variance*36) && (divisor * 4 >= INTERVALS * 3))
+ || variance <= 400) {
+ return avg;
+ }
+ }
+
+ /*
+ * If we have outliers to the upside in our distribution, discard
+ * those by setting the threshold to exclude these outliers, then
+ * calculate the average and standard deviation again. Once we get
+ * down to the bottom 3/4 of our samples, stop excluding samples.
+ *
+ * This can deal with workloads that have long pauses interspersed
+ * with sporadic activity with a bunch of short pauses.
+ */
+ if ((divisor * 4) <= INTERVALS * 3)
+ return UINT_MAX;
+
+ thresh = max - 1;
+ goto again;
+}
+
+/**
+ * menu_select - selects the next idle state to enter
+ * @drv: cpuidle driver containing state data
+ * @dev: the CPU
+ * @stop_tick: indication on whether or not to stop the tick
+ */
+static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
+ bool *stop_tick)
+{
+ struct menu_device *data = this_cpu_ptr(&menu_devices);
+ s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
+ u64 predicted_ns;
+ u64 interactivity_req;
+ unsigned int nr_iowaiters;
+ ktime_t delta, delta_tick;
+ int i, idx;
+
+ if (data->needs_update) {
+ menu_update(drv, dev);
+ data->needs_update = 0;
+ }
+
+ nr_iowaiters = nr_iowait_cpu(dev->cpu);
+
+ /* Find the shortest expected idle interval. */
+ predicted_ns = get_typical_interval(data) * NSEC_PER_USEC;
+ if (predicted_ns > RESIDENCY_THRESHOLD_NS) {
+ unsigned int timer_us;
+
+ /* Determine the time till the closest timer. */
+ delta = tick_nohz_get_sleep_length(&delta_tick);
+ if (unlikely(delta < 0)) {
+ delta = 0;
+ delta_tick = 0;
+ }
+
+ data->next_timer_ns = delta;
+ data->bucket = which_bucket(data->next_timer_ns, nr_iowaiters);
+
+ /* Round up the result for half microseconds. */
+ timer_us = div_u64((RESOLUTION * DECAY * NSEC_PER_USEC) / 2 +
+ data->next_timer_ns *
+ data->correction_factor[data->bucket],
+ RESOLUTION * DECAY * NSEC_PER_USEC);
+ /* Use the lowest expected idle interval to pick the idle state. */
+ predicted_ns = min((u64)timer_us * NSEC_PER_USEC, predicted_ns);
+ } else {
+ /*
+ * Because the next timer event is not going to be determined
+ * in this case, assume that without the tick the closest timer
+ * will be in distant future and that the closest tick will occur
+ * after 1/2 of the tick period.
+ */
+ data->next_timer_ns = KTIME_MAX;
+ delta_tick = TICK_NSEC / 2;
+ data->bucket = which_bucket(KTIME_MAX, nr_iowaiters);
+ }
+
+ if (unlikely(drv->state_count <= 1 || latency_req == 0) ||
+ ((data->next_timer_ns < drv->states[1].target_residency_ns ||
+ latency_req < drv->states[1].exit_latency_ns) &&
+ !dev->states_usage[0].disable)) {
+ /*
+ * In this case state[0] will be used no matter what, so return
+ * it right away and keep the tick running if state[0] is a
+ * polling one.
+ */
+ *stop_tick = !(drv->states[0].flags & CPUIDLE_FLAG_POLLING);
+ return 0;
+ }
+
+ if (tick_nohz_tick_stopped()) {
+ /*
+ * If the tick is already stopped, the cost of possible short
+ * idle duration misprediction is much higher, because the CPU
+ * may be stuck in a shallow idle state for a long time as a
+ * result of it. In that case say we might mispredict and use
+ * the known time till the closest timer event for the idle
+ * state selection.
+ */
+ if (predicted_ns < TICK_NSEC)
+ predicted_ns = data->next_timer_ns;
+ } else {
+ /*
+ * Use the performance multiplier and the user-configurable
+ * latency_req to determine the maximum exit latency.
+ */
+ interactivity_req = div64_u64(predicted_ns,
+ performance_multiplier(nr_iowaiters));
+ if (latency_req > interactivity_req)
+ latency_req = interactivity_req;
+ }
+
+ /*
+ * Find the idle state with the lowest power while satisfying
+ * our constraints.
+ */
+ idx = -1;
+ for (i = 0; i < drv->state_count; i++) {
+ struct cpuidle_state *s = &drv->states[i];
+
+ if (dev->states_usage[i].disable)
+ continue;
+
+ if (idx == -1)
+ idx = i; /* first enabled state */
+
+ if (s->target_residency_ns > predicted_ns) {
+ /*
+ * Use a physical idle state, not busy polling, unless
+ * a timer is going to trigger soon enough.
+ */
+ if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) &&
+ s->exit_latency_ns <= latency_req &&
+ s->target_residency_ns <= data->next_timer_ns) {
+ predicted_ns = s->target_residency_ns;
+ idx = i;
+ break;
+ }
+ if (predicted_ns < TICK_NSEC)
+ break;
+
+ if (!tick_nohz_tick_stopped()) {
+ /*
+ * If the state selected so far is shallow,
+ * waking up early won't hurt, so retain the
+ * tick in that case and let the governor run
+ * again in the next iteration of the loop.
+ */
+ predicted_ns = drv->states[idx].target_residency_ns;
+ break;
+ }
+
+ /*
+ * If the state selected so far is shallow and this
+ * state's target residency matches the time till the
+ * closest timer event, select this one to avoid getting
+ * stuck in the shallow one for too long.
+ */
+ if (drv->states[idx].target_residency_ns < TICK_NSEC &&
+ s->target_residency_ns <= delta_tick)
+ idx = i;
+
+ return idx;
+ }
+ if (s->exit_latency_ns > latency_req)
+ break;
+
+ idx = i;
+ }
+
+ if (idx == -1)
+ idx = 0; /* No states enabled. Must use 0. */
+
+ /*
+ * Don't stop the tick if the selected state is a polling one or if the
+ * expected idle duration is shorter than the tick period length.
+ */
+ if (((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) ||
+ predicted_ns < TICK_NSEC) && !tick_nohz_tick_stopped()) {
+ *stop_tick = false;
+
+ if (idx > 0 && drv->states[idx].target_residency_ns > delta_tick) {
+ /*
+ * The tick is not going to be stopped and the target
+ * residency of the state to be returned is not within
+ * the time until the next timer event including the
+ * tick, so try to correct that.
+ */
+ for (i = idx - 1; i >= 0; i--) {
+ if (dev->states_usage[i].disable)
+ continue;
+
+ idx = i;
+ if (drv->states[i].target_residency_ns <= delta_tick)
+ break;
+ }
+ }
+ }
+
+ return idx;
+}
+
+/**
+ * menu_reflect - records that data structures need update
+ * @dev: the CPU
+ * @index: the index of actual entered state
+ *
+ * NOTE: it's important to be fast here because this operation will add to
+ * the overall exit latency.
+ */
+static void menu_reflect(struct cpuidle_device *dev, int index)
+{
+ struct menu_device *data = this_cpu_ptr(&menu_devices);
+
+ dev->last_state_idx = index;
+ data->needs_update = 1;
+ data->tick_wakeup = tick_nohz_idle_got_tick();
+}
+
+/**
+ * menu_update - attempts to guess what happened after entry
+ * @drv: cpuidle driver containing state data
+ * @dev: the CPU
+ */
+static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
+{
+ struct menu_device *data = this_cpu_ptr(&menu_devices);
+ int last_idx = dev->last_state_idx;
+ struct cpuidle_state *target = &drv->states[last_idx];
+ u64 measured_ns;
+ unsigned int new_factor;
+
+ /*
+ * Try to figure out how much time passed between entry to low
+ * power state and occurrence of the wakeup event.
+ *
+ * If the entered idle state didn't support residency measurements,
+ * we use them anyway if they are short, and if long,
+ * truncate to the whole expected time.
+ *
+ * Any measured amount of time will include the exit latency.
+ * Since we are interested in when the wakeup begun, not when it
+ * was completed, we must subtract the exit latency. However, if
+ * the measured amount of time is less than the exit latency,
+ * assume the state was never reached and the exit latency is 0.
+ */
+
+ if (data->tick_wakeup && data->next_timer_ns > TICK_NSEC) {
+ /*
+ * The nohz code said that there wouldn't be any events within
+ * the tick boundary (if the tick was stopped), but the idle
+ * duration predictor had a differing opinion. Since the CPU
+ * was woken up by a tick (that wasn't stopped after all), the
+ * predictor was not quite right, so assume that the CPU could
+ * have been idle long (but not forever) to help the idle
+ * duration predictor do a better job next time.
+ */
+ measured_ns = 9 * MAX_INTERESTING / 10;
+ } else if ((drv->states[last_idx].flags & CPUIDLE_FLAG_POLLING) &&
+ dev->poll_time_limit) {
+ /*
+ * The CPU exited the "polling" state due to a time limit, so
+ * the idle duration prediction leading to the selection of that
+ * state was inaccurate. If a better prediction had been made,
+ * the CPU might have been woken up from idle by the next timer.
+ * Assume that to be the case.
+ */
+ measured_ns = data->next_timer_ns;
+ } else {
+ /* measured value */
+ measured_ns = dev->last_residency_ns;
+
+ /* Deduct exit latency */
+ if (measured_ns > 2 * target->exit_latency_ns)
+ measured_ns -= target->exit_latency_ns;
+ else
+ measured_ns /= 2;
+ }
+
+ /* Make sure our coefficients do not exceed unity */
+ if (measured_ns > data->next_timer_ns)
+ measured_ns = data->next_timer_ns;
+
+ /* Update our correction ratio */
+ new_factor = data->correction_factor[data->bucket];
+ new_factor -= new_factor / DECAY;
+
+ if (data->next_timer_ns > 0 && measured_ns < MAX_INTERESTING)
+ new_factor += div64_u64(RESOLUTION * measured_ns,
+ data->next_timer_ns);
+ else
+ /*
+ * we were idle so long that we count it as a perfect
+ * prediction
+ */
+ new_factor += RESOLUTION;
+
+ /*
+ * We don't want 0 as factor; we always want at least
+ * a tiny bit of estimated time. Fortunately, due to rounding,
+ * new_factor will stay nonzero regardless of measured_us values
+ * and the compiler can eliminate this test as long as DECAY > 1.
+ */
+ if (DECAY == 1 && unlikely(new_factor == 0))
+ new_factor = 1;
+
+ data->correction_factor[data->bucket] = new_factor;
+
+ /* update the repeating-pattern data */
+ data->intervals[data->interval_ptr++] = ktime_to_us(measured_ns);
+ if (data->interval_ptr >= INTERVALS)
+ data->interval_ptr = 0;
+}
+
+/**
+ * menu_enable_device - scans a CPU's states and does setup
+ * @drv: cpuidle driver
+ * @dev: the CPU
+ */
+static int menu_enable_device(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev)
+{
+ struct menu_device *data = &per_cpu(menu_devices, dev->cpu);
+ int i;
+
+ memset(data, 0, sizeof(struct menu_device));
+
+ /*
+ * if the correction factor is 0 (eg first time init or cpu hotplug
+ * etc), we actually want to start out with a unity factor.
+ */
+ for(i = 0; i < BUCKETS; i++)
+ data->correction_factor[i] = RESOLUTION * DECAY;
+
+ return 0;
+}
+
+static struct cpuidle_governor menu_governor = {
+ .name = "menu",
+ .rating = 20,
+ .enable = menu_enable_device,
+ .select = menu_select,
+ .reflect = menu_reflect,
+};
+
+/**
+ * init_menu - initializes the governor
+ */
+static int __init init_menu(void)
+{
+ return cpuidle_register_governor(&menu_governor);
+}
+
+postcore_initcall(init_menu);
diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c
new file mode 100644
index 0000000000..7244f71c59
--- /dev/null
+++ b/drivers/cpuidle/governors/teo.c
@@ -0,0 +1,695 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Timer events oriented CPU idle governor
+ *
+ * TEO governor:
+ * Copyright (C) 2018 - 2021 Intel Corporation
+ * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
+ *
+ * Util-awareness mechanism:
+ * Copyright (C) 2022 Arm Ltd.
+ * Author: Kajetan Puchalski <kajetan.puchalski@arm.com>
+ */
+
+/**
+ * DOC: teo-description
+ *
+ * The idea of this governor is based on the observation that on many systems
+ * timer events are two or more orders of magnitude more frequent than any
+ * other interrupts, so they are likely to be the most significant cause of CPU
+ * wakeups from idle states. Moreover, information about what happened in the
+ * (relatively recent) past can be used to estimate whether or not the deepest
+ * idle state with target residency within the (known) time till the closest
+ * timer event, referred to as the sleep length, is likely to be suitable for
+ * the upcoming CPU idle period and, if not, then which of the shallower idle
+ * states to choose instead of it.
+ *
+ * Of course, non-timer wakeup sources are more important in some use cases
+ * which can be covered by taking a few most recent idle time intervals of the
+ * CPU into account. However, even in that context it is not necessary to
+ * consider idle duration values greater than the sleep length, because the
+ * closest timer will ultimately wake up the CPU anyway unless it is woken up
+ * earlier.
+ *
+ * Thus this governor estimates whether or not the prospective idle duration of
+ * a CPU is likely to be significantly shorter than the sleep length and selects
+ * an idle state for it accordingly.
+ *
+ * The computations carried out by this governor are based on using bins whose
+ * boundaries are aligned with the target residency parameter values of the CPU
+ * idle states provided by the %CPUIdle driver in the ascending order. That is,
+ * the first bin spans from 0 up to, but not including, the target residency of
+ * the second idle state (idle state 1), the second bin spans from the target
+ * residency of idle state 1 up to, but not including, the target residency of
+ * idle state 2, the third bin spans from the target residency of idle state 2
+ * up to, but not including, the target residency of idle state 3 and so on.
+ * The last bin spans from the target residency of the deepest idle state
+ * supplied by the driver to infinity.
+ *
+ * Two metrics called "hits" and "intercepts" are associated with each bin.
+ * They are updated every time before selecting an idle state for the given CPU
+ * in accordance with what happened last time.
+ *
+ * The "hits" metric reflects the relative frequency of situations in which the
+ * sleep length and the idle duration measured after CPU wakeup fall into the
+ * same bin (that is, the CPU appears to wake up "on time" relative to the sleep
+ * length). In turn, the "intercepts" metric reflects the relative frequency of
+ * situations in which the measured idle duration is so much shorter than the
+ * sleep length that the bin it falls into corresponds to an idle state
+ * shallower than the one whose bin is fallen into by the sleep length (these
+ * situations are referred to as "intercepts" below).
+ *
+ * In addition to the metrics described above, the governor counts recent
+ * intercepts (that is, intercepts that have occurred during the last
+ * %NR_RECENT invocations of it for the given CPU) for each bin.
+ *
+ * In order to select an idle state for a CPU, the governor takes the following
+ * steps (modulo the possible latency constraint that must be taken into account
+ * too):
+ *
+ * 1. Find the deepest CPU idle state whose target residency does not exceed
+ * the current sleep length (the candidate idle state) and compute 3 sums as
+ * follows:
+ *
+ * - The sum of the "hits" and "intercepts" metrics for the candidate state
+ * and all of the deeper idle states (it represents the cases in which the
+ * CPU was idle long enough to avoid being intercepted if the sleep length
+ * had been equal to the current one).
+ *
+ * - The sum of the "intercepts" metrics for all of the idle states shallower
+ * than the candidate one (it represents the cases in which the CPU was not
+ * idle long enough to avoid being intercepted if the sleep length had been
+ * equal to the current one).
+ *
+ * - The sum of the numbers of recent intercepts for all of the idle states
+ * shallower than the candidate one.
+ *
+ * 2. If the second sum is greater than the first one or the third sum is
+ * greater than %NR_RECENT / 2, the CPU is likely to wake up early, so look
+ * for an alternative idle state to select.
+ *
+ * - Traverse the idle states shallower than the candidate one in the
+ * descending order.
+ *
+ * - For each of them compute the sum of the "intercepts" metrics and the sum
+ * of the numbers of recent intercepts over all of the idle states between
+ * it and the candidate one (including the former and excluding the
+ * latter).
+ *
+ * - If each of these sums that needs to be taken into account (because the
+ * check related to it has indicated that the CPU is likely to wake up
+ * early) is greater than a half of the corresponding sum computed in step
+ * 1 (which means that the target residency of the state in question had
+ * not exceeded the idle duration in over a half of the relevant cases),
+ * select the given idle state instead of the candidate one.
+ *
+ * 3. By default, select the candidate state.
+ *
+ * Util-awareness mechanism:
+ *
+ * The idea behind the util-awareness extension is that there are two distinct
+ * scenarios for the CPU which should result in two different approaches to idle
+ * state selection - utilized and not utilized.
+ *
+ * In this case, 'utilized' means that the average runqueue util of the CPU is
+ * above a certain threshold.
+ *
+ * When the CPU is utilized while going into idle, more likely than not it will
+ * be woken up to do more work soon and so a shallower idle state should be
+ * selected to minimise latency and maximise performance. When the CPU is not
+ * being utilized, the usual metrics-based approach to selecting the deepest
+ * available idle state should be preferred to take advantage of the power
+ * saving.
+ *
+ * In order to achieve this, the governor uses a utilization threshold.
+ * The threshold is computed per-CPU as a percentage of the CPU's capacity
+ * by bit shifting the capacity value. Based on testing, the shift of 6 (~1.56%)
+ * seems to be getting the best results.
+ *
+ * Before selecting the next idle state, the governor compares the current CPU
+ * util to the precomputed util threshold. If it's below, it defaults to the
+ * TEO metrics mechanism. If it's above, the closest shallower idle state will
+ * be selected instead, as long as is not a polling state.
+ */
+
+#include <linux/cpuidle.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/sched/clock.h>
+#include <linux/sched/topology.h>
+#include <linux/tick.h>
+
+#include "gov.h"
+
+/*
+ * The number of bits to shift the CPU's capacity by in order to determine
+ * the utilized threshold.
+ *
+ * 6 was chosen based on testing as the number that achieved the best balance
+ * of power and performance on average.
+ *
+ * The resulting threshold is high enough to not be triggered by background
+ * noise and low enough to react quickly when activity starts to ramp up.
+ */
+#define UTIL_THRESHOLD_SHIFT 6
+
+/*
+ * The PULSE value is added to metrics when they grow and the DECAY_SHIFT value
+ * is used for decreasing metrics on a regular basis.
+ */
+#define PULSE 1024
+#define DECAY_SHIFT 3
+
+/*
+ * Number of the most recent idle duration values to take into consideration for
+ * the detection of recent early wakeup patterns.
+ */
+#define NR_RECENT 9
+
+/**
+ * struct teo_bin - Metrics used by the TEO cpuidle governor.
+ * @intercepts: The "intercepts" metric.
+ * @hits: The "hits" metric.
+ * @recent: The number of recent "intercepts".
+ */
+struct teo_bin {
+ unsigned int intercepts;
+ unsigned int hits;
+ unsigned int recent;
+};
+
+/**
+ * struct teo_cpu - CPU data used by the TEO cpuidle governor.
+ * @time_span_ns: Time between idle state selection and post-wakeup update.
+ * @sleep_length_ns: Time till the closest timer event (at the selection time).
+ * @state_bins: Idle state data bins for this CPU.
+ * @total: Grand total of the "intercepts" and "hits" metrics for all bins.
+ * @next_recent_idx: Index of the next @recent_idx entry to update.
+ * @recent_idx: Indices of bins corresponding to recent "intercepts".
+ * @tick_hits: Number of "hits" after TICK_NSEC.
+ * @util_threshold: Threshold above which the CPU is considered utilized
+ */
+struct teo_cpu {
+ s64 time_span_ns;
+ s64 sleep_length_ns;
+ struct teo_bin state_bins[CPUIDLE_STATE_MAX];
+ unsigned int total;
+ int next_recent_idx;
+ int recent_idx[NR_RECENT];
+ unsigned int tick_hits;
+ unsigned long util_threshold;
+};
+
+static DEFINE_PER_CPU(struct teo_cpu, teo_cpus);
+
+/**
+ * teo_cpu_is_utilized - Check if the CPU's util is above the threshold
+ * @cpu: Target CPU
+ * @cpu_data: Governor CPU data for the target CPU
+ */
+#ifdef CONFIG_SMP
+static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data)
+{
+ return sched_cpu_util(cpu) > cpu_data->util_threshold;
+}
+#else
+static bool teo_cpu_is_utilized(int cpu, struct teo_cpu *cpu_data)
+{
+ return false;
+}
+#endif
+
+/**
+ * teo_update - Update CPU metrics after wakeup.
+ * @drv: cpuidle driver containing state data.
+ * @dev: Target CPU.
+ */
+static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
+{
+ struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
+ int i, idx_timer = 0, idx_duration = 0;
+ s64 target_residency_ns;
+ u64 measured_ns;
+
+ if (cpu_data->time_span_ns >= cpu_data->sleep_length_ns) {
+ /*
+ * One of the safety nets has triggered or the wakeup was close
+ * enough to the closest timer event expected at the idle state
+ * selection time to be discarded.
+ */
+ measured_ns = U64_MAX;
+ } else {
+ u64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns;
+
+ /*
+ * The computations below are to determine whether or not the
+ * (saved) time till the next timer event and the measured idle
+ * duration fall into the same "bin", so use last_residency_ns
+ * for that instead of time_span_ns which includes the cpuidle
+ * overhead.
+ */
+ measured_ns = dev->last_residency_ns;
+ /*
+ * The delay between the wakeup and the first instruction
+ * executed by the CPU is not likely to be worst-case every
+ * time, so take 1/2 of the exit latency as a very rough
+ * approximation of the average of it.
+ */
+ if (measured_ns >= lat_ns)
+ measured_ns -= lat_ns / 2;
+ else
+ measured_ns /= 2;
+ }
+
+ cpu_data->total = 0;
+
+ /*
+ * Decay the "hits" and "intercepts" metrics for all of the bins and
+ * find the bins that the sleep length and the measured idle duration
+ * fall into.
+ */
+ for (i = 0; i < drv->state_count; i++) {
+ struct teo_bin *bin = &cpu_data->state_bins[i];
+
+ bin->hits -= bin->hits >> DECAY_SHIFT;
+ bin->intercepts -= bin->intercepts >> DECAY_SHIFT;
+
+ cpu_data->total += bin->hits + bin->intercepts;
+
+ target_residency_ns = drv->states[i].target_residency_ns;
+
+ if (target_residency_ns <= cpu_data->sleep_length_ns) {
+ idx_timer = i;
+ if (target_residency_ns <= measured_ns)
+ idx_duration = i;
+ }
+ }
+
+ i = cpu_data->next_recent_idx++;
+ if (cpu_data->next_recent_idx >= NR_RECENT)
+ cpu_data->next_recent_idx = 0;
+
+ if (cpu_data->recent_idx[i] >= 0)
+ cpu_data->state_bins[cpu_data->recent_idx[i]].recent--;
+
+ /*
+ * If the deepest state's target residency is below the tick length,
+ * make a record of it to help teo_select() decide whether or not
+ * to stop the tick. This effectively adds an extra hits-only bin
+ * beyond the last state-related one.
+ */
+ if (target_residency_ns < TICK_NSEC) {
+ cpu_data->tick_hits -= cpu_data->tick_hits >> DECAY_SHIFT;
+
+ cpu_data->total += cpu_data->tick_hits;
+
+ if (TICK_NSEC <= cpu_data->sleep_length_ns) {
+ idx_timer = drv->state_count;
+ if (TICK_NSEC <= measured_ns) {
+ cpu_data->tick_hits += PULSE;
+ goto end;
+ }
+ }
+ }
+
+ /*
+ * If the measured idle duration falls into the same bin as the sleep
+ * length, this is a "hit", so update the "hits" metric for that bin.
+ * Otherwise, update the "intercepts" metric for the bin fallen into by
+ * the measured idle duration.
+ */
+ if (idx_timer == idx_duration) {
+ cpu_data->state_bins[idx_timer].hits += PULSE;
+ cpu_data->recent_idx[i] = -1;
+ } else {
+ cpu_data->state_bins[idx_duration].intercepts += PULSE;
+ cpu_data->state_bins[idx_duration].recent++;
+ cpu_data->recent_idx[i] = idx_duration;
+ }
+
+end:
+ cpu_data->total += PULSE;
+}
+
+static bool teo_state_ok(int i, struct cpuidle_driver *drv)
+{
+ return !tick_nohz_tick_stopped() ||
+ drv->states[i].target_residency_ns >= TICK_NSEC;
+}
+
+/**
+ * teo_find_shallower_state - Find shallower idle state matching given duration.
+ * @drv: cpuidle driver containing state data.
+ * @dev: Target CPU.
+ * @state_idx: Index of the capping idle state.
+ * @duration_ns: Idle duration value to match.
+ * @no_poll: Don't consider polling states.
+ */
+static int teo_find_shallower_state(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev, int state_idx,
+ s64 duration_ns, bool no_poll)
+{
+ int i;
+
+ for (i = state_idx - 1; i >= 0; i--) {
+ if (dev->states_usage[i].disable ||
+ (no_poll && drv->states[i].flags & CPUIDLE_FLAG_POLLING))
+ continue;
+
+ state_idx = i;
+ if (drv->states[i].target_residency_ns <= duration_ns)
+ break;
+ }
+ return state_idx;
+}
+
+/**
+ * teo_select - Selects the next idle state to enter.
+ * @drv: cpuidle driver containing state data.
+ * @dev: Target CPU.
+ * @stop_tick: Indication on whether or not to stop the scheduler tick.
+ */
+static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
+ bool *stop_tick)
+{
+ struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
+ s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
+ ktime_t delta_tick = TICK_NSEC / 2;
+ unsigned int tick_intercept_sum = 0;
+ unsigned int idx_intercept_sum = 0;
+ unsigned int intercept_sum = 0;
+ unsigned int idx_recent_sum = 0;
+ unsigned int recent_sum = 0;
+ unsigned int idx_hit_sum = 0;
+ unsigned int hit_sum = 0;
+ int constraint_idx = 0;
+ int idx0 = 0, idx = -1;
+ bool alt_intercepts, alt_recent;
+ bool cpu_utilized;
+ s64 duration_ns;
+ int i;
+
+ if (dev->last_state_idx >= 0) {
+ teo_update(drv, dev);
+ dev->last_state_idx = -1;
+ }
+
+ cpu_data->time_span_ns = local_clock();
+ /*
+ * Set the expected sleep length to infinity in case of an early
+ * return.
+ */
+ cpu_data->sleep_length_ns = KTIME_MAX;
+
+ /* Check if there is any choice in the first place. */
+ if (drv->state_count < 2) {
+ idx = 0;
+ goto out_tick;
+ }
+
+ if (!dev->states_usage[0].disable)
+ idx = 0;
+
+ cpu_utilized = teo_cpu_is_utilized(dev->cpu, cpu_data);
+ /*
+ * If the CPU is being utilized over the threshold and there are only 2
+ * states to choose from, the metrics need not be considered, so choose
+ * the shallowest non-polling state and exit.
+ */
+ if (drv->state_count < 3 && cpu_utilized) {
+ /*
+ * If state 0 is enabled and it is not a polling one, select it
+ * right away unless the scheduler tick has been stopped, in
+ * which case care needs to be taken to leave the CPU in a deep
+ * enough state in case it is not woken up any time soon after
+ * all. If state 1 is disabled, though, state 0 must be used
+ * anyway.
+ */
+ if ((!idx && !(drv->states[0].flags & CPUIDLE_FLAG_POLLING) &&
+ teo_state_ok(0, drv)) || dev->states_usage[1].disable) {
+ idx = 0;
+ goto out_tick;
+ }
+ /* Assume that state 1 is not a polling one and use it. */
+ idx = 1;
+ duration_ns = drv->states[1].target_residency_ns;
+ goto end;
+ }
+
+ /* Compute the sums of metrics for early wakeup pattern detection. */
+ for (i = 1; i < drv->state_count; i++) {
+ struct teo_bin *prev_bin = &cpu_data->state_bins[i-1];
+ struct cpuidle_state *s = &drv->states[i];
+
+ /*
+ * Update the sums of idle state mertics for all of the states
+ * shallower than the current one.
+ */
+ intercept_sum += prev_bin->intercepts;
+ hit_sum += prev_bin->hits;
+ recent_sum += prev_bin->recent;
+
+ if (dev->states_usage[i].disable)
+ continue;
+
+ if (idx < 0)
+ idx0 = i; /* first enabled state */
+
+ idx = i;
+
+ if (s->exit_latency_ns <= latency_req)
+ constraint_idx = i;
+
+ /* Save the sums for the current state. */
+ idx_intercept_sum = intercept_sum;
+ idx_hit_sum = hit_sum;
+ idx_recent_sum = recent_sum;
+ }
+
+ /* Avoid unnecessary overhead. */
+ if (idx < 0) {
+ idx = 0; /* No states enabled, must use 0. */
+ goto out_tick;
+ }
+
+ if (idx == idx0) {
+ /*
+ * Only one idle state is enabled, so use it, but do not
+ * allow the tick to be stopped it is shallow enough.
+ */
+ duration_ns = drv->states[idx].target_residency_ns;
+ goto end;
+ }
+
+ tick_intercept_sum = intercept_sum +
+ cpu_data->state_bins[drv->state_count-1].intercepts;
+
+ /*
+ * If the sum of the intercepts metric for all of the idle states
+ * shallower than the current candidate one (idx) is greater than the
+ * sum of the intercepts and hits metrics for the candidate state and
+ * all of the deeper states, or the sum of the numbers of recent
+ * intercepts over all of the states shallower than the candidate one
+ * is greater than a half of the number of recent events taken into
+ * account, a shallower idle state is likely to be a better choice.
+ */
+ alt_intercepts = 2 * idx_intercept_sum > cpu_data->total - idx_hit_sum;
+ alt_recent = idx_recent_sum > NR_RECENT / 2;
+ if (alt_recent || alt_intercepts) {
+ int first_suitable_idx = idx;
+
+ /*
+ * Look for the deepest idle state whose target residency had
+ * not exceeded the idle duration in over a half of the relevant
+ * cases (both with respect to intercepts overall and with
+ * respect to the recent intercepts only) in the past.
+ *
+ * Take the possible duration limitation present if the tick
+ * has been stopped already into account.
+ */
+ intercept_sum = 0;
+ recent_sum = 0;
+
+ for (i = idx - 1; i >= 0; i--) {
+ struct teo_bin *bin = &cpu_data->state_bins[i];
+
+ intercept_sum += bin->intercepts;
+ recent_sum += bin->recent;
+
+ if ((!alt_recent || 2 * recent_sum > idx_recent_sum) &&
+ (!alt_intercepts ||
+ 2 * intercept_sum > idx_intercept_sum)) {
+ /*
+ * Use the current state unless it is too
+ * shallow or disabled, in which case take the
+ * first enabled state that is deep enough.
+ */
+ if (teo_state_ok(i, drv) &&
+ !dev->states_usage[i].disable)
+ idx = i;
+ else
+ idx = first_suitable_idx;
+
+ break;
+ }
+
+ if (dev->states_usage[i].disable)
+ continue;
+
+ if (!teo_state_ok(i, drv)) {
+ /*
+ * The current state is too shallow, but if an
+ * alternative candidate state has been found,
+ * it may still turn out to be a better choice.
+ */
+ if (first_suitable_idx != idx)
+ continue;
+
+ break;
+ }
+
+ first_suitable_idx = i;
+ }
+ }
+
+ /*
+ * If there is a latency constraint, it may be necessary to select an
+ * idle state shallower than the current candidate one.
+ */
+ if (idx > constraint_idx)
+ idx = constraint_idx;
+
+ /*
+ * If the CPU is being utilized over the threshold, choose a shallower
+ * non-polling state to improve latency, unless the scheduler tick has
+ * been stopped already and the shallower state's target residency is
+ * not sufficiently large.
+ */
+ if (cpu_utilized) {
+ i = teo_find_shallower_state(drv, dev, idx, KTIME_MAX, true);
+ if (teo_state_ok(i, drv))
+ idx = i;
+ }
+
+ /*
+ * Skip the timers check if state 0 is the current candidate one,
+ * because an immediate non-timer wakeup is expected in that case.
+ */
+ if (!idx)
+ goto out_tick;
+
+ /*
+ * If state 0 is a polling one, check if the target residency of
+ * the current candidate state is low enough and skip the timers
+ * check in that case too.
+ */
+ if ((drv->states[0].flags & CPUIDLE_FLAG_POLLING) &&
+ drv->states[idx].target_residency_ns < RESIDENCY_THRESHOLD_NS)
+ goto out_tick;
+
+ duration_ns = tick_nohz_get_sleep_length(&delta_tick);
+ cpu_data->sleep_length_ns = duration_ns;
+
+ /*
+ * If the closest expected timer is before the terget residency of the
+ * candidate state, a shallower one needs to be found.
+ */
+ if (drv->states[idx].target_residency_ns > duration_ns) {
+ i = teo_find_shallower_state(drv, dev, idx, duration_ns, false);
+ if (teo_state_ok(i, drv))
+ idx = i;
+ }
+
+ /*
+ * If the selected state's target residency is below the tick length
+ * and intercepts occurring before the tick length are the majority of
+ * total wakeup events, do not stop the tick.
+ */
+ if (drv->states[idx].target_residency_ns < TICK_NSEC &&
+ tick_intercept_sum > cpu_data->total / 2 + cpu_data->total / 8)
+ duration_ns = TICK_NSEC / 2;
+
+end:
+ /*
+ * Allow the tick to be stopped unless the selected state is a polling
+ * one or the expected idle duration is shorter than the tick period
+ * length.
+ */
+ if ((!(drv->states[idx].flags & CPUIDLE_FLAG_POLLING) &&
+ duration_ns >= TICK_NSEC) || tick_nohz_tick_stopped())
+ return idx;
+
+ /*
+ * The tick is not going to be stopped, so if the target residency of
+ * the state to be returned is not within the time till the closest
+ * timer including the tick, try to correct that.
+ */
+ if (idx > idx0 &&
+ drv->states[idx].target_residency_ns > delta_tick)
+ idx = teo_find_shallower_state(drv, dev, idx, delta_tick, false);
+
+out_tick:
+ *stop_tick = false;
+ return idx;
+}
+
+/**
+ * teo_reflect - Note that governor data for the CPU need to be updated.
+ * @dev: Target CPU.
+ * @state: Entered state.
+ */
+static void teo_reflect(struct cpuidle_device *dev, int state)
+{
+ struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
+
+ dev->last_state_idx = state;
+ /*
+ * If the wakeup was not "natural", but triggered by one of the safety
+ * nets, assume that the CPU might have been idle for the entire sleep
+ * length time.
+ */
+ if (dev->poll_time_limit ||
+ (tick_nohz_idle_got_tick() && cpu_data->sleep_length_ns > TICK_NSEC)) {
+ dev->poll_time_limit = false;
+ cpu_data->time_span_ns = cpu_data->sleep_length_ns;
+ } else {
+ cpu_data->time_span_ns = local_clock() - cpu_data->time_span_ns;
+ }
+}
+
+/**
+ * teo_enable_device - Initialize the governor's data for the target CPU.
+ * @drv: cpuidle driver (not used).
+ * @dev: Target CPU.
+ */
+static int teo_enable_device(struct cpuidle_driver *drv,
+ struct cpuidle_device *dev)
+{
+ struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu);
+ unsigned long max_capacity = arch_scale_cpu_capacity(dev->cpu);
+ int i;
+
+ memset(cpu_data, 0, sizeof(*cpu_data));
+ cpu_data->util_threshold = max_capacity >> UTIL_THRESHOLD_SHIFT;
+
+ for (i = 0; i < NR_RECENT; i++)
+ cpu_data->recent_idx[i] = -1;
+
+ return 0;
+}
+
+static struct cpuidle_governor teo_governor = {
+ .name = "teo",
+ .rating = 19,
+ .enable = teo_enable_device,
+ .select = teo_select,
+ .reflect = teo_reflect,
+};
+
+static int __init teo_governor_init(void)
+{
+ return cpuidle_register_governor(&teo_governor);
+}
+
+postcore_initcall(teo_governor_init);
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-12 08:09:31 +0000