diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_pmu.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_pmu.c | 1184 |
1 files changed, 1184 insertions, 0 deletions
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c new file mode 100644 index 000000000..3c9ac6649 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -0,0 +1,1184 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright © 2017-2018 Intel Corporation + */ + +#include <linux/irq.h> +#include <linux/pm_runtime.h> + +#include "gt/intel_engine.h" +#include "gt/intel_engine_pm.h" +#include "gt/intel_engine_user.h" +#include "gt/intel_gt_pm.h" +#include "gt/intel_rc6.h" +#include "gt/intel_rps.h" + +#include "i915_drv.h" +#include "i915_pmu.h" +#include "intel_pm.h" + +/* Frequency for the sampling timer for events which need it. */ +#define FREQUENCY 200 +#define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY) + +#define ENGINE_SAMPLE_MASK \ + (BIT(I915_SAMPLE_BUSY) | \ + BIT(I915_SAMPLE_WAIT) | \ + BIT(I915_SAMPLE_SEMA)) + +#define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS) + +static cpumask_t i915_pmu_cpumask; + +static u8 engine_config_sample(u64 config) +{ + return config & I915_PMU_SAMPLE_MASK; +} + +static u8 engine_event_sample(struct perf_event *event) +{ + return engine_config_sample(event->attr.config); +} + +static u8 engine_event_class(struct perf_event *event) +{ + return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff; +} + +static u8 engine_event_instance(struct perf_event *event) +{ + return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff; +} + +static bool is_engine_config(u64 config) +{ + return config < __I915_PMU_OTHER(0); +} + +static unsigned int config_enabled_bit(u64 config) +{ + if (is_engine_config(config)) + return engine_config_sample(config); + else + return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0)); +} + +static u64 config_enabled_mask(u64 config) +{ + return BIT_ULL(config_enabled_bit(config)); +} + +static bool is_engine_event(struct perf_event *event) +{ + return is_engine_config(event->attr.config); +} + +static unsigned int event_enabled_bit(struct perf_event *event) +{ + return config_enabled_bit(event->attr.config); +} + +static bool pmu_needs_timer(struct i915_pmu *pmu, bool gpu_active) +{ + struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); + u64 enable; + + /* + * Only some counters need the sampling timer. + * + * We start with a bitmask of all currently enabled events. + */ + enable = pmu->enable; + + /* + * Mask out all the ones which do not need the timer, or in + * other words keep all the ones that could need the timer. + */ + enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) | + config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) | + ENGINE_SAMPLE_MASK; + + /* + * When the GPU is idle per-engine counters do not need to be + * running so clear those bits out. + */ + if (!gpu_active) + enable &= ~ENGINE_SAMPLE_MASK; + /* + * Also there is software busyness tracking available we do not + * need the timer for I915_SAMPLE_BUSY counter. + */ + else if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS) + enable &= ~BIT(I915_SAMPLE_BUSY); + + /* + * If some bits remain it means we need the sampling timer running. + */ + return enable; +} + +static u64 __get_rc6(struct intel_gt *gt) +{ + struct drm_i915_private *i915 = gt->i915; + u64 val; + + val = intel_rc6_residency_ns(>->rc6, + IS_VALLEYVIEW(i915) ? + VLV_GT_RENDER_RC6 : + GEN6_GT_GFX_RC6); + + if (HAS_RC6p(i915)) + val += intel_rc6_residency_ns(>->rc6, GEN6_GT_GFX_RC6p); + + if (HAS_RC6pp(i915)) + val += intel_rc6_residency_ns(>->rc6, GEN6_GT_GFX_RC6pp); + + return val; +} + +#if IS_ENABLED(CONFIG_PM) + +static inline s64 ktime_since(const ktime_t kt) +{ + return ktime_to_ns(ktime_sub(ktime_get(), kt)); +} + +static u64 get_rc6(struct intel_gt *gt) +{ + struct drm_i915_private *i915 = gt->i915; + struct i915_pmu *pmu = &i915->pmu; + unsigned long flags; + bool awake = false; + u64 val; + + if (intel_gt_pm_get_if_awake(gt)) { + val = __get_rc6(gt); + intel_gt_pm_put_async(gt); + awake = true; + } + + spin_lock_irqsave(&pmu->lock, flags); + + if (awake) { + pmu->sample[__I915_SAMPLE_RC6].cur = val; + } else { + /* + * We think we are runtime suspended. + * + * Report the delta from when the device was suspended to now, + * on top of the last known real value, as the approximated RC6 + * counter value. + */ + val = ktime_since(pmu->sleep_last); + val += pmu->sample[__I915_SAMPLE_RC6].cur; + } + + if (val < pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur) + val = pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur; + else + pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur = val; + + spin_unlock_irqrestore(&pmu->lock, flags); + + return val; +} + +static void init_rc6(struct i915_pmu *pmu) +{ + struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); + intel_wakeref_t wakeref; + + with_intel_runtime_pm(i915->gt.uncore->rpm, wakeref) { + pmu->sample[__I915_SAMPLE_RC6].cur = __get_rc6(&i915->gt); + pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur = + pmu->sample[__I915_SAMPLE_RC6].cur; + pmu->sleep_last = ktime_get(); + } +} + +static void park_rc6(struct drm_i915_private *i915) +{ + struct i915_pmu *pmu = &i915->pmu; + + pmu->sample[__I915_SAMPLE_RC6].cur = __get_rc6(&i915->gt); + pmu->sleep_last = ktime_get(); +} + +#else + +static u64 get_rc6(struct intel_gt *gt) +{ + return __get_rc6(gt); +} + +static void init_rc6(struct i915_pmu *pmu) { } +static void park_rc6(struct drm_i915_private *i915) {} + +#endif + +static void __i915_pmu_maybe_start_timer(struct i915_pmu *pmu) +{ + if (!pmu->timer_enabled && pmu_needs_timer(pmu, true)) { + pmu->timer_enabled = true; + pmu->timer_last = ktime_get(); + hrtimer_start_range_ns(&pmu->timer, + ns_to_ktime(PERIOD), 0, + HRTIMER_MODE_REL_PINNED); + } +} + +void i915_pmu_gt_parked(struct drm_i915_private *i915) +{ + struct i915_pmu *pmu = &i915->pmu; + + if (!pmu->base.event_init) + return; + + spin_lock_irq(&pmu->lock); + + park_rc6(i915); + + /* + * Signal sampling timer to stop if only engine events are enabled and + * GPU went idle. + */ + pmu->timer_enabled = pmu_needs_timer(pmu, false); + + spin_unlock_irq(&pmu->lock); +} + +void i915_pmu_gt_unparked(struct drm_i915_private *i915) +{ + struct i915_pmu *pmu = &i915->pmu; + + if (!pmu->base.event_init) + return; + + spin_lock_irq(&pmu->lock); + + /* + * Re-enable sampling timer when GPU goes active. + */ + __i915_pmu_maybe_start_timer(pmu); + + spin_unlock_irq(&pmu->lock); +} + +static void +add_sample(struct i915_pmu_sample *sample, u32 val) +{ + sample->cur += val; +} + +static bool exclusive_mmio_access(const struct drm_i915_private *i915) +{ + /* + * We have to avoid concurrent mmio cache line access on gen7 or + * risk a machine hang. For a fun history lesson dig out the old + * userspace intel_gpu_top and run it on Ivybridge or Haswell! + */ + return IS_GEN(i915, 7); +} + +static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns) +{ + struct intel_engine_pmu *pmu = &engine->pmu; + bool busy; + u32 val; + + val = ENGINE_READ_FW(engine, RING_CTL); + if (val == 0) /* powerwell off => engine idle */ + return; + + if (val & RING_WAIT) + add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns); + if (val & RING_WAIT_SEMAPHORE) + add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns); + + /* No need to sample when busy stats are supported. */ + if (intel_engine_supports_stats(engine)) + return; + + /* + * While waiting on a semaphore or event, MI_MODE reports the + * ring as idle. However, previously using the seqno, and with + * execlists sampling, we account for the ring waiting as the + * engine being busy. Therefore, we record the sample as being + * busy if either waiting or !idle. + */ + busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT); + if (!busy) { + val = ENGINE_READ_FW(engine, RING_MI_MODE); + busy = !(val & MODE_IDLE); + } + if (busy) + add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns); +} + +static void +engines_sample(struct intel_gt *gt, unsigned int period_ns) +{ + struct drm_i915_private *i915 = gt->i915; + struct intel_engine_cs *engine; + enum intel_engine_id id; + unsigned long flags; + + if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0) + return; + + if (!intel_gt_pm_is_awake(gt)) + return; + + for_each_engine(engine, gt, id) { + if (!intel_engine_pm_get_if_awake(engine)) + continue; + + if (exclusive_mmio_access(i915)) { + spin_lock_irqsave(&engine->uncore->lock, flags); + engine_sample(engine, period_ns); + spin_unlock_irqrestore(&engine->uncore->lock, flags); + } else { + engine_sample(engine, period_ns); + } + + intel_engine_pm_put_async(engine); + } +} + +static void +add_sample_mult(struct i915_pmu_sample *sample, u32 val, u32 mul) +{ + sample->cur += mul_u32_u32(val, mul); +} + +static bool frequency_sampling_enabled(struct i915_pmu *pmu) +{ + return pmu->enable & + (config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) | + config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)); +} + +static void +frequency_sample(struct intel_gt *gt, unsigned int period_ns) +{ + struct drm_i915_private *i915 = gt->i915; + struct intel_uncore *uncore = gt->uncore; + struct i915_pmu *pmu = &i915->pmu; + struct intel_rps *rps = >->rps; + + if (!frequency_sampling_enabled(pmu)) + return; + + /* Report 0/0 (actual/requested) frequency while parked. */ + if (!intel_gt_pm_get_if_awake(gt)) + return; + + if (pmu->enable & config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) { + u32 val; + + /* + * We take a quick peek here without using forcewake + * so that we don't perturb the system under observation + * (forcewake => !rc6 => increased power use). We expect + * that if the read fails because it is outside of the + * mmio power well, then it will return 0 -- in which + * case we assume the system is running at the intended + * frequency. Fortunately, the read should rarely fail! + */ + val = intel_uncore_read_fw(uncore, GEN6_RPSTAT1); + if (val) + val = intel_rps_get_cagf(rps, val); + else + val = rps->cur_freq; + + add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT], + intel_gpu_freq(rps, val), period_ns / 1000); + } + + if (pmu->enable & config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) { + add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ], + intel_gpu_freq(rps, rps->cur_freq), + period_ns / 1000); + } + + intel_gt_pm_put_async(gt); +} + +static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer) +{ + struct drm_i915_private *i915 = + container_of(hrtimer, struct drm_i915_private, pmu.timer); + struct i915_pmu *pmu = &i915->pmu; + struct intel_gt *gt = &i915->gt; + unsigned int period_ns; + ktime_t now; + + if (!READ_ONCE(pmu->timer_enabled)) + return HRTIMER_NORESTART; + + now = ktime_get(); + period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last)); + pmu->timer_last = now; + + /* + * Strictly speaking the passed in period may not be 100% accurate for + * all internal calculation, since some amount of time can be spent on + * grabbing the forcewake. However the potential error from timer call- + * back delay greatly dominates this so we keep it simple. + */ + engines_sample(gt, period_ns); + frequency_sample(gt, period_ns); + + hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD)); + + return HRTIMER_RESTART; +} + +static u64 count_interrupts(struct drm_i915_private *i915) +{ + /* open-coded kstat_irqs() */ + struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq); + u64 sum = 0; + int cpu; + + if (!desc || !desc->kstat_irqs) + return 0; + + for_each_possible_cpu(cpu) + sum += *per_cpu_ptr(desc->kstat_irqs, cpu); + + return sum; +} + +static void i915_pmu_event_destroy(struct perf_event *event) +{ + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + + drm_WARN_ON(&i915->drm, event->parent); +} + +static int +engine_event_status(struct intel_engine_cs *engine, + enum drm_i915_pmu_engine_sample sample) +{ + switch (sample) { + case I915_SAMPLE_BUSY: + case I915_SAMPLE_WAIT: + break; + case I915_SAMPLE_SEMA: + if (INTEL_GEN(engine->i915) < 6) + return -ENODEV; + break; + default: + return -ENOENT; + } + + return 0; +} + +static int +config_status(struct drm_i915_private *i915, u64 config) +{ + switch (config) { + case I915_PMU_ACTUAL_FREQUENCY: + if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) + /* Requires a mutex for sampling! */ + return -ENODEV; + fallthrough; + case I915_PMU_REQUESTED_FREQUENCY: + if (INTEL_GEN(i915) < 6) + return -ENODEV; + break; + case I915_PMU_INTERRUPTS: + break; + case I915_PMU_RC6_RESIDENCY: + if (!HAS_RC6(i915)) + return -ENODEV; + break; + default: + return -ENOENT; + } + + return 0; +} + +static int engine_event_init(struct perf_event *event) +{ + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + struct intel_engine_cs *engine; + + engine = intel_engine_lookup_user(i915, engine_event_class(event), + engine_event_instance(event)); + if (!engine) + return -ENODEV; + + return engine_event_status(engine, engine_event_sample(event)); +} + +static int i915_pmu_event_init(struct perf_event *event) +{ + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + int ret; + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + /* unsupported modes and filters */ + if (event->attr.sample_period) /* no sampling */ + return -EINVAL; + + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + if (event->cpu < 0) + return -EINVAL; + + /* only allow running on one cpu at a time */ + if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask)) + return -EINVAL; + + if (is_engine_event(event)) + ret = engine_event_init(event); + else + ret = config_status(i915, event->attr.config); + if (ret) + return ret; + + if (!event->parent) + event->destroy = i915_pmu_event_destroy; + + return 0; +} + +static u64 __i915_pmu_event_read(struct perf_event *event) +{ + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + struct i915_pmu *pmu = &i915->pmu; + u64 val = 0; + + if (is_engine_event(event)) { + u8 sample = engine_event_sample(event); + struct intel_engine_cs *engine; + + engine = intel_engine_lookup_user(i915, + engine_event_class(event), + engine_event_instance(event)); + + if (drm_WARN_ON_ONCE(&i915->drm, !engine)) { + /* Do nothing */ + } else if (sample == I915_SAMPLE_BUSY && + intel_engine_supports_stats(engine)) { + ktime_t unused; + + val = ktime_to_ns(intel_engine_get_busy_time(engine, + &unused)); + } else { + val = engine->pmu.sample[sample].cur; + } + } else { + switch (event->attr.config) { + case I915_PMU_ACTUAL_FREQUENCY: + val = + div_u64(pmu->sample[__I915_SAMPLE_FREQ_ACT].cur, + USEC_PER_SEC /* to MHz */); + break; + case I915_PMU_REQUESTED_FREQUENCY: + val = + div_u64(pmu->sample[__I915_SAMPLE_FREQ_REQ].cur, + USEC_PER_SEC /* to MHz */); + break; + case I915_PMU_INTERRUPTS: + val = count_interrupts(i915); + break; + case I915_PMU_RC6_RESIDENCY: + val = get_rc6(&i915->gt); + break; + } + } + + return val; +} + +static void i915_pmu_event_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + u64 prev, new; + +again: + prev = local64_read(&hwc->prev_count); + new = __i915_pmu_event_read(event); + + if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev) + goto again; + + local64_add(new - prev, &event->count); +} + +static void i915_pmu_enable(struct perf_event *event) +{ + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + unsigned int bit = event_enabled_bit(event); + struct i915_pmu *pmu = &i915->pmu; + unsigned long flags; + + spin_lock_irqsave(&pmu->lock, flags); + + /* + * Update the bitmask of enabled events and increment + * the event reference counter. + */ + BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS); + GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count)); + GEM_BUG_ON(pmu->enable_count[bit] == ~0); + + pmu->enable |= BIT_ULL(bit); + pmu->enable_count[bit]++; + + /* + * Start the sampling timer if needed and not already enabled. + */ + __i915_pmu_maybe_start_timer(pmu); + + /* + * For per-engine events the bitmask and reference counting + * is stored per engine. + */ + if (is_engine_event(event)) { + u8 sample = engine_event_sample(event); + struct intel_engine_cs *engine; + + engine = intel_engine_lookup_user(i915, + engine_event_class(event), + engine_event_instance(event)); + + BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) != + I915_ENGINE_SAMPLE_COUNT); + BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) != + I915_ENGINE_SAMPLE_COUNT); + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); + GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0); + + engine->pmu.enable |= BIT(sample); + engine->pmu.enable_count[sample]++; + } + + spin_unlock_irqrestore(&pmu->lock, flags); + + /* + * Store the current counter value so we can report the correct delta + * for all listeners. Even when the event was already enabled and has + * an existing non-zero value. + */ + local64_set(&event->hw.prev_count, __i915_pmu_event_read(event)); +} + +static void i915_pmu_disable(struct perf_event *event) +{ + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + unsigned int bit = event_enabled_bit(event); + struct i915_pmu *pmu = &i915->pmu; + unsigned long flags; + + spin_lock_irqsave(&pmu->lock, flags); + + if (is_engine_event(event)) { + u8 sample = engine_event_sample(event); + struct intel_engine_cs *engine; + + engine = intel_engine_lookup_user(i915, + engine_event_class(event), + engine_event_instance(event)); + + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); + GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); + GEM_BUG_ON(engine->pmu.enable_count[sample] == 0); + + /* + * Decrement the reference count and clear the enabled + * bitmask when the last listener on an event goes away. + */ + if (--engine->pmu.enable_count[sample] == 0) + engine->pmu.enable &= ~BIT(sample); + } + + GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count)); + GEM_BUG_ON(pmu->enable_count[bit] == 0); + /* + * Decrement the reference count and clear the enabled + * bitmask when the last listener on an event goes away. + */ + if (--pmu->enable_count[bit] == 0) { + pmu->enable &= ~BIT_ULL(bit); + pmu->timer_enabled &= pmu_needs_timer(pmu, true); + } + + spin_unlock_irqrestore(&pmu->lock, flags); +} + +static void i915_pmu_event_start(struct perf_event *event, int flags) +{ + i915_pmu_enable(event); + event->hw.state = 0; +} + +static void i915_pmu_event_stop(struct perf_event *event, int flags) +{ + if (flags & PERF_EF_UPDATE) + i915_pmu_event_read(event); + i915_pmu_disable(event); + event->hw.state = PERF_HES_STOPPED; +} + +static int i915_pmu_event_add(struct perf_event *event, int flags) +{ + if (flags & PERF_EF_START) + i915_pmu_event_start(event, flags); + + return 0; +} + +static void i915_pmu_event_del(struct perf_event *event, int flags) +{ + i915_pmu_event_stop(event, PERF_EF_UPDATE); +} + +static int i915_pmu_event_event_idx(struct perf_event *event) +{ + return 0; +} + +struct i915_str_attribute { + struct device_attribute attr; + const char *str; +}; + +static ssize_t i915_pmu_format_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct i915_str_attribute *eattr; + + eattr = container_of(attr, struct i915_str_attribute, attr); + return sprintf(buf, "%s\n", eattr->str); +} + +#define I915_PMU_FORMAT_ATTR(_name, _config) \ + (&((struct i915_str_attribute[]) { \ + { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \ + .str = _config, } \ + })[0].attr.attr) + +static struct attribute *i915_pmu_format_attrs[] = { + I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"), + NULL, +}; + +static const struct attribute_group i915_pmu_format_attr_group = { + .name = "format", + .attrs = i915_pmu_format_attrs, +}; + +struct i915_ext_attribute { + struct device_attribute attr; + unsigned long val; +}; + +static ssize_t i915_pmu_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct i915_ext_attribute *eattr; + + eattr = container_of(attr, struct i915_ext_attribute, attr); + return sprintf(buf, "config=0x%lx\n", eattr->val); +} + +static ssize_t +i915_pmu_get_attr_cpumask(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask); +} + +static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL); + +static struct attribute *i915_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL, +}; + +static const struct attribute_group i915_pmu_cpumask_attr_group = { + .attrs = i915_cpumask_attrs, +}; + +#define __event(__config, __name, __unit) \ +{ \ + .config = (__config), \ + .name = (__name), \ + .unit = (__unit), \ +} + +#define __engine_event(__sample, __name) \ +{ \ + .sample = (__sample), \ + .name = (__name), \ +} + +static struct i915_ext_attribute * +add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config) +{ + sysfs_attr_init(&attr->attr.attr); + attr->attr.attr.name = name; + attr->attr.attr.mode = 0444; + attr->attr.show = i915_pmu_event_show; + attr->val = config; + + return ++attr; +} + +static struct perf_pmu_events_attr * +add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name, + const char *str) +{ + sysfs_attr_init(&attr->attr.attr); + attr->attr.attr.name = name; + attr->attr.attr.mode = 0444; + attr->attr.show = perf_event_sysfs_show; + attr->event_str = str; + + return ++attr; +} + +static struct attribute ** +create_event_attributes(struct i915_pmu *pmu) +{ + struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); + static const struct { + u64 config; + const char *name; + const char *unit; + } events[] = { + __event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "M"), + __event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "M"), + __event(I915_PMU_INTERRUPTS, "interrupts", NULL), + __event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"), + }; + static const struct { + enum drm_i915_pmu_engine_sample sample; + char *name; + } engine_events[] = { + __engine_event(I915_SAMPLE_BUSY, "busy"), + __engine_event(I915_SAMPLE_SEMA, "sema"), + __engine_event(I915_SAMPLE_WAIT, "wait"), + }; + unsigned int count = 0; + struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter; + struct i915_ext_attribute *i915_attr = NULL, *i915_iter; + struct attribute **attr = NULL, **attr_iter; + struct intel_engine_cs *engine; + unsigned int i; + + /* Count how many counters we will be exposing. */ + for (i = 0; i < ARRAY_SIZE(events); i++) { + if (!config_status(i915, events[i].config)) + count++; + } + + for_each_uabi_engine(engine, i915) { + for (i = 0; i < ARRAY_SIZE(engine_events); i++) { + if (!engine_event_status(engine, + engine_events[i].sample)) + count++; + } + } + + /* Allocate attribute objects and table. */ + i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL); + if (!i915_attr) + goto err_alloc; + + pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL); + if (!pmu_attr) + goto err_alloc; + + /* Max one pointer of each attribute type plus a termination entry. */ + attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL); + if (!attr) + goto err_alloc; + + i915_iter = i915_attr; + pmu_iter = pmu_attr; + attr_iter = attr; + + /* Initialize supported non-engine counters. */ + for (i = 0; i < ARRAY_SIZE(events); i++) { + char *str; + + if (config_status(i915, events[i].config)) + continue; + + str = kstrdup(events[i].name, GFP_KERNEL); + if (!str) + goto err; + + *attr_iter++ = &i915_iter->attr.attr; + i915_iter = add_i915_attr(i915_iter, str, events[i].config); + + if (events[i].unit) { + str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name); + if (!str) + goto err; + + *attr_iter++ = &pmu_iter->attr.attr; + pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit); + } + } + + /* Initialize supported engine counters. */ + for_each_uabi_engine(engine, i915) { + for (i = 0; i < ARRAY_SIZE(engine_events); i++) { + char *str; + + if (engine_event_status(engine, + engine_events[i].sample)) + continue; + + str = kasprintf(GFP_KERNEL, "%s-%s", + engine->name, engine_events[i].name); + if (!str) + goto err; + + *attr_iter++ = &i915_iter->attr.attr; + i915_iter = + add_i915_attr(i915_iter, str, + __I915_PMU_ENGINE(engine->uabi_class, + engine->uabi_instance, + engine_events[i].sample)); + + str = kasprintf(GFP_KERNEL, "%s-%s.unit", + engine->name, engine_events[i].name); + if (!str) + goto err; + + *attr_iter++ = &pmu_iter->attr.attr; + pmu_iter = add_pmu_attr(pmu_iter, str, "ns"); + } + } + + pmu->i915_attr = i915_attr; + pmu->pmu_attr = pmu_attr; + + return attr; + +err:; + for (attr_iter = attr; *attr_iter; attr_iter++) + kfree((*attr_iter)->name); + +err_alloc: + kfree(attr); + kfree(i915_attr); + kfree(pmu_attr); + + return NULL; +} + +static void free_event_attributes(struct i915_pmu *pmu) +{ + struct attribute **attr_iter = pmu->events_attr_group.attrs; + + for (; *attr_iter; attr_iter++) + kfree((*attr_iter)->name); + + kfree(pmu->events_attr_group.attrs); + kfree(pmu->i915_attr); + kfree(pmu->pmu_attr); + + pmu->events_attr_group.attrs = NULL; + pmu->i915_attr = NULL; + pmu->pmu_attr = NULL; +} + +static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) +{ + struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); + + GEM_BUG_ON(!pmu->base.event_init); + + /* Select the first online CPU as a designated reader. */ + if (!cpumask_weight(&i915_pmu_cpumask)) + cpumask_set_cpu(cpu, &i915_pmu_cpumask); + + return 0; +} + +static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) +{ + struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); + unsigned int target; + + GEM_BUG_ON(!pmu->base.event_init); + + if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) { + target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); + /* Migrate events if there is a valid target */ + if (target < nr_cpu_ids) { + cpumask_set_cpu(target, &i915_pmu_cpumask); + perf_pmu_migrate_context(&pmu->base, cpu, target); + } + } + + return 0; +} + +static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu) +{ + enum cpuhp_state slot; + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, + "perf/x86/intel/i915:online", + i915_pmu_cpu_online, + i915_pmu_cpu_offline); + if (ret < 0) + return ret; + + slot = ret; + ret = cpuhp_state_add_instance(slot, &pmu->cpuhp.node); + if (ret) { + cpuhp_remove_multi_state(slot); + return ret; + } + + pmu->cpuhp.slot = slot; + return 0; +} + +static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu) +{ + struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); + + drm_WARN_ON(&i915->drm, pmu->cpuhp.slot == CPUHP_INVALID); + drm_WARN_ON(&i915->drm, cpuhp_state_remove_instance(pmu->cpuhp.slot, &pmu->cpuhp.node)); + cpuhp_remove_multi_state(pmu->cpuhp.slot); + pmu->cpuhp.slot = CPUHP_INVALID; +} + +static bool is_igp(struct drm_i915_private *i915) +{ + struct pci_dev *pdev = i915->drm.pdev; + + /* IGP is 0000:00:02.0 */ + return pci_domain_nr(pdev->bus) == 0 && + pdev->bus->number == 0 && + PCI_SLOT(pdev->devfn) == 2 && + PCI_FUNC(pdev->devfn) == 0; +} + +void i915_pmu_register(struct drm_i915_private *i915) +{ + struct i915_pmu *pmu = &i915->pmu; + const struct attribute_group *attr_groups[] = { + &i915_pmu_format_attr_group, + &pmu->events_attr_group, + &i915_pmu_cpumask_attr_group, + NULL + }; + + int ret = -ENOMEM; + + if (INTEL_GEN(i915) <= 2) { + drm_info(&i915->drm, "PMU not supported for this GPU."); + return; + } + + spin_lock_init(&pmu->lock); + hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + pmu->timer.function = i915_sample; + pmu->cpuhp.slot = CPUHP_INVALID; + init_rc6(pmu); + + if (!is_igp(i915)) { + pmu->name = kasprintf(GFP_KERNEL, + "i915_%s", + dev_name(i915->drm.dev)); + if (pmu->name) { + /* tools/perf reserves colons as special. */ + strreplace((char *)pmu->name, ':', '_'); + } + } else { + pmu->name = "i915"; + } + if (!pmu->name) + goto err; + + pmu->events_attr_group.name = "events"; + pmu->events_attr_group.attrs = create_event_attributes(pmu); + if (!pmu->events_attr_group.attrs) + goto err_name; + + pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups), + GFP_KERNEL); + if (!pmu->base.attr_groups) + goto err_attr; + + pmu->base.module = THIS_MODULE; + pmu->base.task_ctx_nr = perf_invalid_context; + pmu->base.event_init = i915_pmu_event_init; + pmu->base.add = i915_pmu_event_add; + pmu->base.del = i915_pmu_event_del; + pmu->base.start = i915_pmu_event_start; + pmu->base.stop = i915_pmu_event_stop; + pmu->base.read = i915_pmu_event_read; + pmu->base.event_idx = i915_pmu_event_event_idx; + + ret = perf_pmu_register(&pmu->base, pmu->name, -1); + if (ret) + goto err_groups; + + ret = i915_pmu_register_cpuhp_state(pmu); + if (ret) + goto err_unreg; + + return; + +err_unreg: + perf_pmu_unregister(&pmu->base); +err_groups: + kfree(pmu->base.attr_groups); +err_attr: + pmu->base.event_init = NULL; + free_event_attributes(pmu); +err_name: + if (!is_igp(i915)) + kfree(pmu->name); +err: + drm_notice(&i915->drm, "Failed to register PMU!\n"); +} + +void i915_pmu_unregister(struct drm_i915_private *i915) +{ + struct i915_pmu *pmu = &i915->pmu; + + if (!pmu->base.event_init) + return; + + drm_WARN_ON(&i915->drm, pmu->enable); + + hrtimer_cancel(&pmu->timer); + + i915_pmu_unregister_cpuhp_state(pmu); + + perf_pmu_unregister(&pmu->base); + pmu->base.event_init = NULL; + kfree(pmu->base.attr_groups); + if (!is_igp(i915)) + kfree(pmu->name); + free_event_attributes(pmu); +} |