// SPDX-License-Identifier: GPL-2.0-only #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include #include #include #include /* * Arbitrary large value chosen to be sufficiently large to minimize noise but * sufficiently small to complete quickly. */ static unsigned int nr_function_calls = 100000; module_param(nr_function_calls, uint, 0); MODULE_PARM_DESC(nr_function_calls, "How many times to call the relevant tracee"); /* * The number of ops associated with a call site affects whether a tracer can * be called directly or whether it's necessary to go via the list func, which * can be significantly more expensive. */ static unsigned int nr_ops_relevant = 1; module_param(nr_ops_relevant, uint, 0); MODULE_PARM_DESC(nr_ops_relevant, "How many ftrace_ops to associate with the relevant tracee"); /* * On architectures where all call sites share the same trampoline, having * tracers enabled for distinct functions can force the use of the list func * and incur overhead for all call sites. */ static unsigned int nr_ops_irrelevant; module_param(nr_ops_irrelevant, uint, 0); MODULE_PARM_DESC(nr_ops_irrelevant, "How many ftrace_ops to associate with the irrelevant tracee"); /* * On architectures with DYNAMIC_FTRACE_WITH_REGS, saving the full pt_regs can * be more expensive than only saving the minimal necessary regs. */ static bool save_regs; module_param(save_regs, bool, 0); MODULE_PARM_DESC(save_regs, "Register ops with FTRACE_OPS_FL_SAVE_REGS (save all registers in the trampoline)"); static bool assist_recursion; module_param(assist_recursion, bool, 0); MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RECURSION"); static bool assist_rcu; module_param(assist_rcu, bool, 0); MODULE_PARM_DESC(assist_reursion, "Register ops with FTRACE_OPS_FL_RCU"); /* * By default, a trivial tracer is used which immediately returns to mimimize * overhead. Sometimes a consistency check using a more expensive tracer is * desireable. */ static bool check_count; module_param(check_count, bool, 0); MODULE_PARM_DESC(check_count, "Check that tracers are called the expected number of times\n"); /* * Usually it's not interesting to leave the ops registered after the test * runs, but sometimes it can be useful to leave them registered so that they * can be inspected through the tracefs 'enabled_functions' file. */ static bool persist; module_param(persist, bool, 0); MODULE_PARM_DESC(persist, "Successfully load module and leave ftrace ops registered after test completes\n"); /* * Marked as noinline to ensure that an out-of-line traceable copy is * generated by the compiler. * * The barrier() ensures the compiler won't elide calls by determining there * are no side-effects. */ static noinline void tracee_relevant(void) { barrier(); } /* * Marked as noinline to ensure that an out-of-line traceable copy is * generated by the compiler. * * The barrier() ensures the compiler won't elide calls by determining there * are no side-effects. */ static noinline void tracee_irrelevant(void) { barrier(); } struct sample_ops { struct ftrace_ops ops; unsigned int count; }; static void ops_func_nop(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { /* do nothing */ } static void ops_func_count(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { struct sample_ops *self; self = container_of(op, struct sample_ops, ops); self->count++; } static struct sample_ops *ops_relevant; static struct sample_ops *ops_irrelevant; static struct sample_ops *ops_alloc_init(void *tracee, ftrace_func_t func, unsigned long flags, int nr) { struct sample_ops *ops; ops = kcalloc(nr, sizeof(*ops), GFP_KERNEL); if (WARN_ON_ONCE(!ops)) return NULL; for (unsigned int i = 0; i < nr; i++) { ops[i].ops.func = func; ops[i].ops.flags = flags; WARN_ON_ONCE(ftrace_set_filter_ip(&ops[i].ops, (unsigned long)tracee, 0, 0)); WARN_ON_ONCE(register_ftrace_function(&ops[i].ops)); } return ops; } static void ops_destroy(struct sample_ops *ops, int nr) { if (!ops) return; for (unsigned int i = 0; i < nr; i++) { WARN_ON_ONCE(unregister_ftrace_function(&ops[i].ops)); ftrace_free_filter(&ops[i].ops); } kfree(ops); } static void ops_check(struct sample_ops *ops, int nr, unsigned int expected_count) { if (!ops || !check_count) return; for (unsigned int i = 0; i < nr; i++) { if (ops->count == expected_count) continue; pr_warn("Counter called %u times (expected %u)\n", ops->count, expected_count); } } static ftrace_func_t tracer_relevant = ops_func_nop; static ftrace_func_t tracer_irrelevant = ops_func_nop; static int __init ftrace_ops_sample_init(void) { unsigned long flags = 0; ktime_t start, end; u64 period; if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && save_regs) { pr_info("this kernel does not support saving registers\n"); save_regs = false; } else if (save_regs) { flags |= FTRACE_OPS_FL_SAVE_REGS; } if (assist_recursion) flags |= FTRACE_OPS_FL_RECURSION; if (assist_rcu) flags |= FTRACE_OPS_FL_RCU; if (check_count) { tracer_relevant = ops_func_count; tracer_irrelevant = ops_func_count; } pr_info("registering:\n" " relevant ops: %u\n" " tracee: %ps\n" " tracer: %ps\n" " irrelevant ops: %u\n" " tracee: %ps\n" " tracer: %ps\n" " saving registers: %s\n" " assist recursion: %s\n" " assist RCU: %s\n", nr_ops_relevant, tracee_relevant, tracer_relevant, nr_ops_irrelevant, tracee_irrelevant, tracer_irrelevant, save_regs ? "YES" : "NO", assist_recursion ? "YES" : "NO", assist_rcu ? "YES" : "NO"); ops_relevant = ops_alloc_init(tracee_relevant, tracer_relevant, flags, nr_ops_relevant); ops_irrelevant = ops_alloc_init(tracee_irrelevant, tracer_irrelevant, flags, nr_ops_irrelevant); start = ktime_get(); for (unsigned int i = 0; i < nr_function_calls; i++) tracee_relevant(); end = ktime_get(); ops_check(ops_relevant, nr_ops_relevant, nr_function_calls); ops_check(ops_irrelevant, nr_ops_irrelevant, 0); period = ktime_to_ns(ktime_sub(end, start)); pr_info("Attempted %u calls to %ps in %lluns (%lluns / call)\n", nr_function_calls, tracee_relevant, period, div_u64(period, nr_function_calls)); if (persist) return 0; ops_destroy(ops_relevant, nr_ops_relevant); ops_destroy(ops_irrelevant, nr_ops_irrelevant); /* * The benchmark completed sucessfully, but there's no reason to keep * the module around. Return an error do the user doesn't have to * manually unload the module. */ return -EINVAL; } module_init(ftrace_ops_sample_init); static void __exit ftrace_ops_sample_exit(void) { ops_destroy(ops_relevant, nr_ops_relevant); ops_destroy(ops_irrelevant, nr_ops_irrelevant); } module_exit(ftrace_ops_sample_exit); MODULE_AUTHOR("Mark Rutland"); MODULE_DESCRIPTION("Example of using custom ftrace_ops"); MODULE_LICENSE("GPL");