Adding upstream version 6.1.76.upstream/6.1.76

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 520 insertions, 0 deletions

diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
new file mode 100644
index 000000000..ecc990ec1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
@@ -0,0 +1,520 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2015-2021 Intel Corporation
+ */
+
+#include <linux/kthread.h>
+#include <linux/string_helpers.h>
+#include <trace/events/dma_fence.h>
+#include <uapi/linux/sched/types.h>
+
+#include "i915_drv.h"
+#include "i915_trace.h"
+#include "intel_breadcrumbs.h"
+#include "intel_context.h"
+#include "intel_engine_pm.h"
+#include "intel_gt_pm.h"
+#include "intel_gt_requests.h"
+
+static bool irq_enable(struct intel_breadcrumbs *b)
+{
+	return intel_engine_irq_enable(b->irq_engine);
+}
+
+static void irq_disable(struct intel_breadcrumbs *b)
+{
+	intel_engine_irq_disable(b->irq_engine);
+}
+
+static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
+{
+	/*
+	 * Since we are waiting on a request, the GPU should be busy
+	 * and should have its own rpm reference.
+	 */
+	if (GEM_WARN_ON(!intel_gt_pm_get_if_awake(b->irq_engine->gt)))
+		return;
+
+	/*
+	 * The breadcrumb irq will be disarmed on the interrupt after the
+	 * waiters are signaled. This gives us a single interrupt window in
+	 * which we can add a new waiter and avoid the cost of re-enabling
+	 * the irq.
+	 */
+	WRITE_ONCE(b->irq_armed, true);
+
+	/* Requests may have completed before we could enable the interrupt. */
+	if (!b->irq_enabled++ && b->irq_enable(b))
+		irq_work_queue(&b->irq_work);
+}
+
+static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
+{
+	if (!b->irq_engine)
+		return;
+
+	spin_lock(&b->irq_lock);
+	if (!b->irq_armed)
+		__intel_breadcrumbs_arm_irq(b);
+	spin_unlock(&b->irq_lock);
+}
+
+static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
+{
+	GEM_BUG_ON(!b->irq_enabled);
+	if (!--b->irq_enabled)
+		b->irq_disable(b);
+
+	WRITE_ONCE(b->irq_armed, false);
+	intel_gt_pm_put_async(b->irq_engine->gt);
+}
+
+static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
+{
+	spin_lock(&b->irq_lock);
+	if (b->irq_armed)
+		__intel_breadcrumbs_disarm_irq(b);
+	spin_unlock(&b->irq_lock);
+}
+
+static void add_signaling_context(struct intel_breadcrumbs *b,
+				  struct intel_context *ce)
+{
+	lockdep_assert_held(&ce->signal_lock);
+
+	spin_lock(&b->signalers_lock);
+	list_add_rcu(&ce->signal_link, &b->signalers);
+	spin_unlock(&b->signalers_lock);
+}
+
+static bool remove_signaling_context(struct intel_breadcrumbs *b,
+				     struct intel_context *ce)
+{
+	lockdep_assert_held(&ce->signal_lock);
+
+	if (!list_empty(&ce->signals))
+		return false;
+
+	spin_lock(&b->signalers_lock);
+	list_del_rcu(&ce->signal_link);
+	spin_unlock(&b->signalers_lock);
+
+	return true;
+}
+
+__maybe_unused static bool
+check_signal_order(struct intel_context *ce, struct i915_request *rq)
+{
+	if (rq->context != ce)
+		return false;
+
+	if (!list_is_last(&rq->signal_link, &ce->signals) &&
+	    i915_seqno_passed(rq->fence.seqno,
+			      list_next_entry(rq, signal_link)->fence.seqno))
+		return false;
+
+	if (!list_is_first(&rq->signal_link, &ce->signals) &&
+	    i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
+			      rq->fence.seqno))
+		return false;
+
+	return true;
+}
+
+static bool
+__dma_fence_signal(struct dma_fence *fence)
+{
+	return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
+}
+
+static void
+__dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
+{
+	fence->timestamp = timestamp;
+	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
+	trace_dma_fence_signaled(fence);
+}
+
+static void
+__dma_fence_signal__notify(struct dma_fence *fence,
+			   const struct list_head *list)
+{
+	struct dma_fence_cb *cur, *tmp;
+
+	lockdep_assert_held(fence->lock);
+
+	list_for_each_entry_safe(cur, tmp, list, node) {
+		INIT_LIST_HEAD(&cur->node);
+		cur->func(fence, cur);
+	}
+}
+
+static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
+{
+	if (b->irq_engine)
+		intel_engine_add_retire(b->irq_engine, tl);
+}
+
+static struct llist_node *
+slist_add(struct llist_node *node, struct llist_node *head)
+{
+	node->next = head;
+	return node;
+}
+
+static void signal_irq_work(struct irq_work *work)
+{
+	struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
+	const ktime_t timestamp = ktime_get();
+	struct llist_node *signal, *sn;
+	struct intel_context *ce;
+
+	signal = NULL;
+	if (unlikely(!llist_empty(&b->signaled_requests)))
+		signal = llist_del_all(&b->signaled_requests);
+
+	/*
+	 * Keep the irq armed until the interrupt after all listeners are gone.
+	 *
+	 * Enabling/disabling the interrupt is rather costly, roughly a couple
+	 * of hundred microseconds. If we are proactive and enable/disable
+	 * the interrupt around every request that wants a breadcrumb, we
+	 * quickly drown in the extra orders of magnitude of latency imposed
+	 * on request submission.
+	 *
+	 * So we try to be lazy, and keep the interrupts enabled until no
+	 * more listeners appear within a breadcrumb interrupt interval (that
+	 * is until a request completes that no one cares about). The
+	 * observation is that listeners come in batches, and will often
+	 * listen to a bunch of requests in succession. Though note on icl+,
+	 * interrupts are always enabled due to concerns with rc6 being
+	 * dysfunctional with per-engine interrupt masking.
+	 *
+	 * We also try to avoid raising too many interrupts, as they may
+	 * be generated by userspace batches and it is unfortunately rather
+	 * too easy to drown the CPU under a flood of GPU interrupts. Thus
+	 * whenever no one appears to be listening, we turn off the interrupts.
+	 * Fewer interrupts should conserve power -- at the very least, fewer
+	 * interrupt draw less ire from other users of the system and tools
+	 * like powertop.
+	 */
+	if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers))
+		intel_breadcrumbs_disarm_irq(b);
+
+	rcu_read_lock();
+	atomic_inc(&b->signaler_active);
+	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
+		struct i915_request *rq;
+
+		list_for_each_entry_rcu(rq, &ce->signals, signal_link) {
+			bool release;
+
+			if (!__i915_request_is_complete(rq))
+				break;
+
+			if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL,
+						&rq->fence.flags))
+				break;
+
+			/*
+			 * Queue for execution after dropping the signaling
+			 * spinlock as the callback chain may end up adding
+			 * more signalers to the same context or engine.
+			 */
+			spin_lock(&ce->signal_lock);
+			list_del_rcu(&rq->signal_link);
+			release = remove_signaling_context(b, ce);
+			spin_unlock(&ce->signal_lock);
+			if (release) {
+				if (intel_timeline_is_last(ce->timeline, rq))
+					add_retire(b, ce->timeline);
+				intel_context_put(ce);
+			}
+
+			if (__dma_fence_signal(&rq->fence))
+				/* We own signal_node now, xfer to local list */
+				signal = slist_add(&rq->signal_node, signal);
+			else
+				i915_request_put(rq);
+		}
+	}
+	atomic_dec(&b->signaler_active);
+	rcu_read_unlock();
+
+	llist_for_each_safe(signal, sn, signal) {
+		struct i915_request *rq =
+			llist_entry(signal, typeof(*rq), signal_node);
+		struct list_head cb_list;
+
+		if (rq->engine->sched_engine->retire_inflight_request_prio)
+			rq->engine->sched_engine->retire_inflight_request_prio(rq);
+
+		spin_lock(&rq->lock);
+		list_replace(&rq->fence.cb_list, &cb_list);
+		__dma_fence_signal__timestamp(&rq->fence, timestamp);
+		__dma_fence_signal__notify(&rq->fence, &cb_list);
+		spin_unlock(&rq->lock);
+
+		i915_request_put(rq);
+	}
+
+	if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers))
+		intel_breadcrumbs_arm_irq(b);
+}
+
+struct intel_breadcrumbs *
+intel_breadcrumbs_create(struct intel_engine_cs *irq_engine)
+{
+	struct intel_breadcrumbs *b;
+
+	b = kzalloc(sizeof(*b), GFP_KERNEL);
+	if (!b)
+		return NULL;
+
+	kref_init(&b->ref);
+
+	spin_lock_init(&b->signalers_lock);
+	INIT_LIST_HEAD(&b->signalers);
+	init_llist_head(&b->signaled_requests);
+
+	spin_lock_init(&b->irq_lock);
+	init_irq_work(&b->irq_work, signal_irq_work);
+
+	b->irq_engine = irq_engine;
+	b->irq_enable = irq_enable;
+	b->irq_disable = irq_disable;
+
+	return b;
+}
+
+void intel_breadcrumbs_reset(struct intel_breadcrumbs *b)
+{
+	unsigned long flags;
+
+	if (!b->irq_engine)
+		return;
+
+	spin_lock_irqsave(&b->irq_lock, flags);
+
+	if (b->irq_enabled)
+		b->irq_enable(b);
+	else
+		b->irq_disable(b);
+
+	spin_unlock_irqrestore(&b->irq_lock, flags);
+}
+
+void __intel_breadcrumbs_park(struct intel_breadcrumbs *b)
+{
+	if (!READ_ONCE(b->irq_armed))
+		return;
+
+	/* Kick the work once more to drain the signalers, and disarm the irq */
+	irq_work_sync(&b->irq_work);
+	while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {
+		local_irq_disable();
+		signal_irq_work(&b->irq_work);
+		local_irq_enable();
+		cond_resched();
+	}
+}
+
+void intel_breadcrumbs_free(struct kref *kref)
+{
+	struct intel_breadcrumbs *b = container_of(kref, typeof(*b), ref);
+
+	irq_work_sync(&b->irq_work);
+	GEM_BUG_ON(!list_empty(&b->signalers));
+	GEM_BUG_ON(b->irq_armed);
+
+	kfree(b);
+}
+
+static void irq_signal_request(struct i915_request *rq,
+			       struct intel_breadcrumbs *b)
+{
+	if (!__dma_fence_signal(&rq->fence))
+		return;
+
+	i915_request_get(rq);
+	if (llist_add(&rq->signal_node, &b->signaled_requests))
+		irq_work_queue(&b->irq_work);
+}
+
+static void insert_breadcrumb(struct i915_request *rq)
+{
+	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;
+	struct intel_context *ce = rq->context;
+	struct list_head *pos;
+
+	if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))
+		return;
+
+	/*
+	 * If the request is already completed, we can transfer it
+	 * straight onto a signaled list, and queue the irq worker for
+	 * its signal completion.
+	 */
+	if (__i915_request_is_complete(rq)) {
+		irq_signal_request(rq, b);
+		return;
+	}
+
+	if (list_empty(&ce->signals)) {
+		intel_context_get(ce);
+		add_signaling_context(b, ce);
+		pos = &ce->signals;
+	} else {
+		/*
+		 * We keep the seqno in retirement order, so we can break
+		 * inside intel_engine_signal_breadcrumbs as soon as we've
+		 * passed the last completed request (or seen a request that
+		 * hasn't event started). We could walk the timeline->requests,
+		 * but keeping a separate signalers_list has the advantage of
+		 * hopefully being much smaller than the full list and so
+		 * provides faster iteration and detection when there are no
+		 * more interrupts required for this context.
+		 *
+		 * We typically expect to add new signalers in order, so we
+		 * start looking for our insertion point from the tail of
+		 * the list.
+		 */
+		list_for_each_prev(pos, &ce->signals) {
+			struct i915_request *it =
+				list_entry(pos, typeof(*it), signal_link);
+
+			if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
+				break;
+		}
+	}
+
+	i915_request_get(rq);
+	list_add_rcu(&rq->signal_link, pos);
+	GEM_BUG_ON(!check_signal_order(ce, rq));
+	GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags));
+	set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
+
+	/*
+	 * Defer enabling the interrupt to after HW submission and recheck
+	 * the request as it may have completed and raised the interrupt as
+	 * we were attaching it into the lists.
+	 */
+	if (!b->irq_armed || __i915_request_is_complete(rq))
+		irq_work_queue(&b->irq_work);
+}
+
+bool i915_request_enable_breadcrumb(struct i915_request *rq)
+{
+	struct intel_context *ce = rq->context;
+
+	/* Serialises with i915_request_retire() using rq->lock */
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
+		return true;
+
+	/*
+	 * Peek at i915_request_submit()/i915_request_unsubmit() status.
+	 *
+	 * If the request is not yet active (and not signaled), we will
+	 * attach the breadcrumb later.
+	 */
+	if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
+		return true;
+
+	spin_lock(&ce->signal_lock);
+	if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
+		insert_breadcrumb(rq);
+	spin_unlock(&ce->signal_lock);
+
+	return true;
+}
+
+void i915_request_cancel_breadcrumb(struct i915_request *rq)
+{
+	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;
+	struct intel_context *ce = rq->context;
+	bool release;
+
+	spin_lock(&ce->signal_lock);
+	if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
+		spin_unlock(&ce->signal_lock);
+		return;
+	}
+
+	list_del_rcu(&rq->signal_link);
+	release = remove_signaling_context(b, ce);
+	spin_unlock(&ce->signal_lock);
+	if (release)
+		intel_context_put(ce);
+
+	if (__i915_request_is_complete(rq))
+		irq_signal_request(rq, b);
+
+	i915_request_put(rq);
+}
+
+void intel_context_remove_breadcrumbs(struct intel_context *ce,
+				      struct intel_breadcrumbs *b)
+{
+	struct i915_request *rq, *rn;
+	bool release = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ce->signal_lock, flags);
+
+	if (list_empty(&ce->signals))
+		goto unlock;
+
+	list_for_each_entry_safe(rq, rn, &ce->signals, signal_link) {
+		GEM_BUG_ON(!__i915_request_is_complete(rq));
+		if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL,
+					&rq->fence.flags))
+			continue;
+
+		list_del_rcu(&rq->signal_link);
+		irq_signal_request(rq, b);
+		i915_request_put(rq);
+	}
+	release = remove_signaling_context(b, ce);
+
+unlock:
+	spin_unlock_irqrestore(&ce->signal_lock, flags);
+	if (release)
+		intel_context_put(ce);
+
+	while (atomic_read(&b->signaler_active))
+		cpu_relax();
+}
+
+static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p)
+{
+	struct intel_context *ce;
+	struct i915_request *rq;
+
+	drm_printf(p, "Signals:\n");
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
+		list_for_each_entry_rcu(rq, &ce->signals, signal_link)
+			drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
+				   rq->fence.context, rq->fence.seqno,
+				   __i915_request_is_complete(rq) ? "!" :
+				   __i915_request_has_started(rq) ? "*" :
+				   "",
+				   jiffies_to_msecs(jiffies - rq->emitted_jiffies));
+	}
+	rcu_read_unlock();
+}
+
+void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
+				    struct drm_printer *p)
+{
+	struct intel_breadcrumbs *b;
+
+	b = engine->breadcrumbs;
+	if (!b)
+		return;
+
+	drm_printf(p, "IRQ: %s\n", str_enabled_disabled(b->irq_armed));
+	if (!list_empty(&b->signalers))
+		print_signals(b, p);
+}