summaryrefslogtreecommitdiffstats
path: root/drivers/powercap/idle_inject.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
commitace9429bb58fd418f0c81d4c2835699bddf6bde6 (patch)
treeb2d64bc10158fdd5497876388cd68142ca374ed3 /drivers/powercap/idle_inject.c
parentInitial commit. (diff)
downloadlinux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.tar.xz
linux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.zip
Adding upstream version 6.6.15.upstream/6.6.15
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'drivers/powercap/idle_inject.c')
-rw-r--r--drivers/powercap/idle_inject.c422
1 files changed, 422 insertions, 0 deletions
diff --git a/drivers/powercap/idle_inject.c b/drivers/powercap/idle_inject.c
new file mode 100644
index 0000000000..e18a2cc4e4
--- /dev/null
+++ b/drivers/powercap/idle_inject.c
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Linaro Limited
+ *
+ * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
+ *
+ * The idle injection framework provides a way to force CPUs to enter idle
+ * states for a specified fraction of time over a specified period.
+ *
+ * It relies on the smpboot kthreads feature providing common code for CPU
+ * hotplug and thread [un]parking.
+ *
+ * All of the kthreads used for idle injection are created at init time.
+ *
+ * Next, the users of the idle injection framework provide a cpumask via
+ * its register function. The kthreads will be synchronized with respect to
+ * this cpumask.
+ *
+ * The idle + run duration is specified via separate helpers and that allows
+ * idle injection to be started.
+ *
+ * The idle injection kthreads will call play_idle_precise() with the idle
+ * duration and max allowed latency specified as per the above.
+ *
+ * After all of them have been woken up, a timer is set to start the next idle
+ * injection cycle.
+ *
+ * The timer interrupt handler will wake up the idle injection kthreads for
+ * all of the CPUs in the cpumask provided by the user.
+ *
+ * Idle injection is stopped synchronously and no leftover idle injection
+ * kthread activity after its completion is guaranteed.
+ *
+ * It is up to the user of this framework to provide a lock for higher-level
+ * synchronization to prevent race conditions like starting idle injection
+ * while unregistering from the framework.
+ */
+#define pr_fmt(fmt) "ii_dev: " fmt
+
+#include <linux/cpu.h>
+#include <linux/hrtimer.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/smpboot.h>
+#include <linux/idle_inject.h>
+
+#include <uapi/linux/sched/types.h>
+
+/**
+ * struct idle_inject_thread - task on/off switch structure
+ * @tsk: task injecting the idle cycles
+ * @should_run: whether or not to run the task (for the smpboot kthread API)
+ */
+struct idle_inject_thread {
+ struct task_struct *tsk;
+ int should_run;
+};
+
+/**
+ * struct idle_inject_device - idle injection data
+ * @timer: idle injection period timer
+ * @idle_duration_us: duration of CPU idle time to inject
+ * @run_duration_us: duration of CPU run time to allow
+ * @latency_us: max allowed latency
+ * @update: Optional callback deciding whether or not to skip idle
+ * injection in the given cycle.
+ * @cpumask: mask of CPUs affected by idle injection
+ *
+ * This structure is used to define per instance idle inject device data. Each
+ * instance has an idle duration, a run duration and mask of CPUs to inject
+ * idle.
+ *
+ * Actual CPU idle time is injected by calling kernel scheduler interface
+ * play_idle_precise(). There is one optional callback that can be registered
+ * by calling idle_inject_register_full():
+ *
+ * update() - This callback is invoked just before waking up CPUs to inject
+ * idle. If it returns false, CPUs are not woken up to inject idle in the given
+ * cycle. It also allows the caller to readjust the idle and run duration by
+ * calling idle_inject_set_duration() for the next cycle.
+ */
+struct idle_inject_device {
+ struct hrtimer timer;
+ unsigned int idle_duration_us;
+ unsigned int run_duration_us;
+ unsigned int latency_us;
+ bool (*update)(void);
+ unsigned long cpumask[];
+};
+
+static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
+static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
+
+/**
+ * idle_inject_wakeup - Wake up idle injection threads
+ * @ii_dev: target idle injection device
+ *
+ * Every idle injection task associated with the given idle injection device
+ * and running on an online CPU will be woken up.
+ */
+static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
+{
+ struct idle_inject_thread *iit;
+ unsigned int cpu;
+
+ for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
+ iit = per_cpu_ptr(&idle_inject_thread, cpu);
+ iit->should_run = 1;
+ wake_up_process(iit->tsk);
+ }
+}
+
+/**
+ * idle_inject_timer_fn - idle injection timer function
+ * @timer: idle injection hrtimer
+ *
+ * This function is called when the idle injection timer expires. It wakes up
+ * idle injection tasks associated with the timer and they, in turn, invoke
+ * play_idle_precise() to inject a specified amount of CPU idle time.
+ *
+ * Return: HRTIMER_RESTART.
+ */
+static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
+{
+ unsigned int duration_us;
+ struct idle_inject_device *ii_dev =
+ container_of(timer, struct idle_inject_device, timer);
+
+ if (!ii_dev->update || (ii_dev->update && ii_dev->update()))
+ idle_inject_wakeup(ii_dev);
+
+ duration_us = READ_ONCE(ii_dev->run_duration_us);
+ duration_us += READ_ONCE(ii_dev->idle_duration_us);
+
+ hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
+
+ return HRTIMER_RESTART;
+}
+
+/**
+ * idle_inject_fn - idle injection work function
+ * @cpu: the CPU owning the task
+ *
+ * This function calls play_idle_precise() to inject a specified amount of CPU
+ * idle time.
+ */
+static void idle_inject_fn(unsigned int cpu)
+{
+ struct idle_inject_device *ii_dev;
+ struct idle_inject_thread *iit;
+
+ ii_dev = per_cpu(idle_inject_device, cpu);
+ iit = per_cpu_ptr(&idle_inject_thread, cpu);
+
+ /*
+ * Let the smpboot main loop know that the task should not run again.
+ */
+ iit->should_run = 0;
+
+ play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
+ READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
+}
+
+/**
+ * idle_inject_set_duration - idle and run duration update helper
+ * @ii_dev: idle injection control device structure
+ * @run_duration_us: CPU run time to allow in microseconds
+ * @idle_duration_us: CPU idle time to inject in microseconds
+ */
+void idle_inject_set_duration(struct idle_inject_device *ii_dev,
+ unsigned int run_duration_us,
+ unsigned int idle_duration_us)
+{
+ if (run_duration_us + idle_duration_us) {
+ WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
+ WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
+ }
+ if (!run_duration_us)
+ pr_debug("CPU is forced to 100 percent idle\n");
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);
+
+/**
+ * idle_inject_get_duration - idle and run duration retrieval helper
+ * @ii_dev: idle injection control device structure
+ * @run_duration_us: memory location to store the current CPU run time
+ * @idle_duration_us: memory location to store the current CPU idle time
+ */
+void idle_inject_get_duration(struct idle_inject_device *ii_dev,
+ unsigned int *run_duration_us,
+ unsigned int *idle_duration_us)
+{
+ *run_duration_us = READ_ONCE(ii_dev->run_duration_us);
+ *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);
+
+/**
+ * idle_inject_set_latency - set the maximum latency allowed
+ * @ii_dev: idle injection control device structure
+ * @latency_us: set the latency requirement for the idle state
+ */
+void idle_inject_set_latency(struct idle_inject_device *ii_dev,
+ unsigned int latency_us)
+{
+ WRITE_ONCE(ii_dev->latency_us, latency_us);
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);
+
+/**
+ * idle_inject_start - start idle injections
+ * @ii_dev: idle injection control device structure
+ *
+ * The function starts idle injection by first waking up all of the idle
+ * injection kthreads associated with @ii_dev to let them inject CPU idle time
+ * sets up a timer to start the next idle injection period.
+ *
+ * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
+ */
+int idle_inject_start(struct idle_inject_device *ii_dev)
+{
+ unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
+ unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
+
+ if (!(idle_duration_us + run_duration_us))
+ return -EINVAL;
+
+ pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
+ cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
+
+ idle_inject_wakeup(ii_dev);
+
+ hrtimer_start(&ii_dev->timer,
+ ns_to_ktime((idle_duration_us + run_duration_us) *
+ NSEC_PER_USEC),
+ HRTIMER_MODE_REL);
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);
+
+/**
+ * idle_inject_stop - stops idle injections
+ * @ii_dev: idle injection control device structure
+ *
+ * The function stops idle injection and waits for the threads to finish work.
+ * If CPU idle time is being injected when this function runs, then it will
+ * wait until the end of the cycle.
+ *
+ * When it returns, there is no more idle injection kthread activity. The
+ * kthreads are scheduled out and the periodic timer is off.
+ */
+void idle_inject_stop(struct idle_inject_device *ii_dev)
+{
+ struct idle_inject_thread *iit;
+ unsigned int cpu;
+
+ pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
+ cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
+
+ hrtimer_cancel(&ii_dev->timer);
+
+ /*
+ * Stopping idle injection requires all of the idle injection kthreads
+ * associated with the given cpumask to be parked and stay that way, so
+ * prevent CPUs from going online at this point. Any CPUs going online
+ * after the loop below will be covered by clearing the should_run flag
+ * that will cause the smpboot main loop to schedule them out.
+ */
+ cpu_hotplug_disable();
+
+ /*
+ * Iterate over all (online + offline) CPUs here in case one of them
+ * goes offline with the should_run flag set so as to prevent its idle
+ * injection kthread from running when the CPU goes online again after
+ * the ii_dev has been freed.
+ */
+ for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
+ iit = per_cpu_ptr(&idle_inject_thread, cpu);
+ iit->should_run = 0;
+
+ wait_task_inactive(iit->tsk, TASK_ANY);
+ }
+
+ cpu_hotplug_enable();
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);
+
+/**
+ * idle_inject_setup - prepare the current task for idle injection
+ * @cpu: not used
+ *
+ * Called once, this function is in charge of setting the current task's
+ * scheduler parameters to make it an RT task.
+ */
+static void idle_inject_setup(unsigned int cpu)
+{
+ sched_set_fifo(current);
+}
+
+/**
+ * idle_inject_should_run - function helper for the smpboot API
+ * @cpu: CPU the kthread is running on
+ *
+ * Return: whether or not the thread can run.
+ */
+static int idle_inject_should_run(unsigned int cpu)
+{
+ struct idle_inject_thread *iit =
+ per_cpu_ptr(&idle_inject_thread, cpu);
+
+ return iit->should_run;
+}
+
+/**
+ * idle_inject_register_full - initialize idle injection on a set of CPUs
+ * @cpumask: CPUs to be affected by idle injection
+ * @update: This callback is called just before waking up CPUs to inject
+ * idle
+ *
+ * This function creates an idle injection control device structure for the
+ * given set of CPUs and initializes the timer associated with it. This
+ * function also allows to register update()callback.
+ * It does not start any injection cycles.
+ *
+ * Return: NULL if memory allocation fails, idle injection control device
+ * pointer on success.
+ */
+
+struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,
+ bool (*update)(void))
+{
+ struct idle_inject_device *ii_dev;
+ int cpu, cpu_rb;
+
+ ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
+ if (!ii_dev)
+ return NULL;
+
+ cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
+ hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ ii_dev->timer.function = idle_inject_timer_fn;
+ ii_dev->latency_us = UINT_MAX;
+ ii_dev->update = update;
+
+ for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
+
+ if (per_cpu(idle_inject_device, cpu)) {
+ pr_err("cpu%d is already registered\n", cpu);
+ goto out_rollback;
+ }
+
+ per_cpu(idle_inject_device, cpu) = ii_dev;
+ }
+
+ return ii_dev;
+
+out_rollback:
+ for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
+ if (cpu == cpu_rb)
+ break;
+ per_cpu(idle_inject_device, cpu_rb) = NULL;
+ }
+
+ kfree(ii_dev);
+
+ return NULL;
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT);
+
+/**
+ * idle_inject_register - initialize idle injection on a set of CPUs
+ * @cpumask: CPUs to be affected by idle injection
+ *
+ * This function creates an idle injection control device structure for the
+ * given set of CPUs and initializes the timer associated with it. It does not
+ * start any injection cycles.
+ *
+ * Return: NULL if memory allocation fails, idle injection control device
+ * pointer on success.
+ */
+struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
+{
+ return idle_inject_register_full(cpumask, NULL);
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);
+
+/**
+ * idle_inject_unregister - unregister idle injection control device
+ * @ii_dev: idle injection control device to unregister
+ *
+ * The function stops idle injection for the given control device,
+ * unregisters its kthreads and frees memory allocated when that device was
+ * created.
+ */
+void idle_inject_unregister(struct idle_inject_device *ii_dev)
+{
+ unsigned int cpu;
+
+ idle_inject_stop(ii_dev);
+
+ for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
+ per_cpu(idle_inject_device, cpu) = NULL;
+
+ kfree(ii_dev);
+}
+EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);
+
+static struct smp_hotplug_thread idle_inject_threads = {
+ .store = &idle_inject_thread.tsk,
+ .setup = idle_inject_setup,
+ .thread_fn = idle_inject_fn,
+ .thread_comm = "idle_inject/%u",
+ .thread_should_run = idle_inject_should_run,
+};
+
+static int __init idle_inject_init(void)
+{
+ return smpboot_register_percpu_thread(&idle_inject_threads);
+}
+early_initcall(idle_inject_init);