diff options
Diffstat (limited to 'drivers/gpu/drm/xe/xe_guc_ct.c')
-rw-r--r-- | drivers/gpu/drm/xe/xe_guc_ct.c | 265 |
1 files changed, 211 insertions, 54 deletions
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 24a33fa364..8bbfa45798 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -9,16 +9,21 @@ #include <linux/circ_buf.h> #include <linux/delay.h> +#include <kunit/static_stub.h> + #include <drm/drm_managed.h> #include "abi/guc_actions_abi.h" +#include "abi/guc_actions_sriov_abi.h" #include "abi/guc_klvs_abi.h" #include "xe_bo.h" #include "xe_device.h" #include "xe_gt.h" #include "xe_gt_pagefault.h" +#include "xe_gt_printk.h" #include "xe_gt_tlb_invalidation.h" #include "xe_guc.h" +#include "xe_guc_relay.h" #include "xe_guc_submit.h" #include "xe_map.h" #include "xe_pm.h" @@ -28,6 +33,7 @@ struct g2h_fence { u32 *response_buffer; u32 seqno; + u32 response_data; u16 response_len; u16 error; u16 hint; @@ -40,6 +46,7 @@ struct g2h_fence { static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) { g2h_fence->response_buffer = response_buffer; + g2h_fence->response_data = 0; g2h_fence->response_len = 0; g2h_fence->fail = false; g2h_fence->retry = false; @@ -113,6 +120,7 @@ static void guc_ct_fini(struct drm_device *drm, void *arg) { struct xe_guc_ct *ct = arg; + destroy_workqueue(ct->g2h_wq); xa_destroy(&ct->fence_lookup); } @@ -138,17 +146,24 @@ int xe_guc_ct_init(struct xe_guc_ct *ct) xe_assert(xe, !(guc_ct_size() % PAGE_SIZE)); - drmm_mutex_init(&xe->drm, &ct->lock); + ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", 0); + if (!ct->g2h_wq) + return -ENOMEM; + spin_lock_init(&ct->fast_lock); xa_init(&ct->fence_lookup); INIT_WORK(&ct->g2h_worker, g2h_worker_func); init_waitqueue_head(&ct->wq); init_waitqueue_head(&ct->g2h_fence_wq); + err = drmm_mutex_init(&xe->drm, &ct->lock); + if (err) + return err; + primelockdep(ct); bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), - XE_BO_CREATE_VRAM_IF_DGFX(tile) | + XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(bo)) return PTR_ERR(bo); @@ -159,6 +174,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct) if (err) return err; + xe_assert(xe, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); + ct->state = XE_GUC_CT_STATE_DISABLED; return 0; } @@ -278,12 +295,35 @@ static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) return ret > 0 ? -EPROTO : ret; } +static void xe_guc_ct_set_state(struct xe_guc_ct *ct, + enum xe_guc_ct_state state) +{ + mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ + spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ + + xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || + state == XE_GUC_CT_STATE_STOPPED); + + ct->g2h_outstanding = 0; + ct->state = state; + + spin_unlock_irq(&ct->fast_lock); + + /* + * Lockdep doesn't like this under the fast lock and he destroy only + * needs to be serialized with the send path which ct lock provides. + */ + xa_destroy(&ct->fence_lookup); + + mutex_unlock(&ct->lock); +} + int xe_guc_ct_enable(struct xe_guc_ct *ct) { struct xe_device *xe = ct_to_xe(ct); int err; - xe_assert(xe, !ct->enabled); + xe_assert(xe, !xe_guc_ct_enabled(ct)); guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); @@ -300,12 +340,7 @@ int xe_guc_ct_enable(struct xe_guc_ct *ct) if (err) goto err_out; - mutex_lock(&ct->lock); - spin_lock_irq(&ct->fast_lock); - ct->g2h_outstanding = 0; - ct->enabled = true; - spin_unlock_irq(&ct->fast_lock); - mutex_unlock(&ct->lock); + xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED); smp_mb(); wake_up_all(&ct->wq); @@ -319,15 +354,34 @@ err_out: return err; } +static void stop_g2h_handler(struct xe_guc_ct *ct) +{ + cancel_work_sync(&ct->g2h_worker); +} + +/** + * xe_guc_ct_disable - Set GuC to disabled state + * @ct: the &xe_guc_ct + * + * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected + * in this transition. + */ void xe_guc_ct_disable(struct xe_guc_ct *ct) { - mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ - spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ - ct->enabled = false; /* Finally disable CT communication */ - spin_unlock_irq(&ct->fast_lock); - mutex_unlock(&ct->lock); + xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED); + stop_g2h_handler(ct); +} - xa_destroy(&ct->fence_lookup); +/** + * xe_guc_ct_stop - Set GuC to stopped state + * @ct: the &xe_guc_ct + * + * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h + */ +void xe_guc_ct_stop(struct xe_guc_ct *ct) +{ + xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED); + stop_g2h_handler(ct); } static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) @@ -448,7 +502,7 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, GUC_HXG_EVENT_MSG_0_DATA0, action[0]); } else { cmd[1] = - FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_EVENT) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) | FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | GUC_HXG_EVENT_MSG_0_DATA0, action[0]); } @@ -475,13 +529,34 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, return 0; } +/* + * The CT protocol accepts a 16 bits fence. This field is fully owned by the + * driver, the GuC will just copy it to the reply message. Since we need to + * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, + * we use one bit of the seqno as an indicator for that and a rolling counter + * for the remaining 15 bits. + */ +#define CT_SEQNO_MASK GENMASK(14, 0) +#define CT_SEQNO_UNTRACKED BIT(15) +static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) +{ + u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; + + if (!is_g2h_fence) + seqno |= CT_SEQNO_UNTRACKED; + + return seqno; +} + static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) { struct xe_device *xe = ct_to_xe(ct); + u16 seqno; int ret; + xe_assert(xe, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED); xe_assert(xe, !g2h_len || !g2h_fence); xe_assert(xe, !num_g2h || !g2h_fence); xe_assert(xe, !g2h_len || num_g2h); @@ -493,11 +568,18 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, goto out; } - if (unlikely(!ct->enabled)) { + if (ct->state == XE_GUC_CT_STATE_DISABLED) { ret = -ENODEV; goto out; } + if (ct->state == XE_GUC_CT_STATE_STOPPED) { + ret = -ECANCELED; + goto out; + } + + xe_assert(xe, xe_guc_ct_enabled(ct)); + if (g2h_fence) { g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; num_g2h = 1; @@ -505,7 +587,7 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, if (g2h_fence_needs_alloc(g2h_fence)) { void *ptr; - g2h_fence->seqno = (ct->fence_seqno++ & 0xffff); + g2h_fence->seqno = next_ct_seqno(ct, true); ptr = xa_store(&ct->fence_lookup, g2h_fence->seqno, g2h_fence, GFP_ATOMIC); @@ -514,6 +596,10 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, goto out; } } + + seqno = g2h_fence->seqno; + } else { + seqno = next_ct_seqno(ct, false); } if (g2h_len) @@ -523,8 +609,7 @@ retry: if (unlikely(ret)) goto out_unlock; - ret = h2g_write(ct, action, len, g2h_fence ? g2h_fence->seqno : 0, - !!g2h_fence); + ret = h2g_write(ct, action, len, seqno, !!g2h_fence); if (unlikely(ret)) { if (ret == -EAGAIN) goto retry; @@ -682,7 +767,8 @@ static bool retry_failure(struct xe_guc_ct *ct, int ret) return false; #define ct_alive(ct) \ - (ct->enabled && !ct->ctbs.h2g.info.broken && !ct->ctbs.g2h.info.broken) + (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ + !ct->ctbs.g2h.info.broken) if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) return false; #undef ct_alive @@ -752,12 +838,31 @@ retry_same_fence: ret = -EIO; } - return ret > 0 ? 0 : ret; + return ret > 0 ? response_buffer ? g2h_fence.response_len : g2h_fence.response_data : ret; } +/** + * xe_guc_ct_send_recv - Send and receive HXG to the GuC + * @ct: the &xe_guc_ct + * @action: the dword array with `HXG Request`_ message (can't be NULL) + * @len: length of the `HXG Request`_ message (in dwords, can't be 0) + * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) + * + * Send a `HXG Request`_ message to the GuC over CT communication channel and + * blocks until GuC replies with a `HXG Response`_ message. + * + * For non-blocking communication with GuC use xe_guc_ct_send(). + * + * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. + * + * Return: response length (in dwords) if &response_buffer was not NULL, or + * DATA0 from `HXG Response`_ if &response_buffer was NULL, or + * a negative error code on failure. + */ int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, u32 *response_buffer) { + KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); return guc_ct_send_recv(ct, action, len, response_buffer, false); } @@ -767,9 +872,20 @@ int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, return guc_ct_send_recv(ct, action, len, response_buffer, true); } +static u32 *msg_to_hxg(u32 *msg) +{ + return msg + GUC_CTB_MSG_MIN_LEN; +} + +static u32 msg_len_to_hxg_len(u32 len) +{ + return len - GUC_CTB_MSG_MIN_LEN; +} + static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) { - u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]); + u32 *hxg = msg_to_hxg(msg); + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); lockdep_assert_held(&ct->lock); @@ -786,18 +902,41 @@ static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) { - struct xe_device *xe = ct_to_xe(ct); - u32 response_len = len - GUC_CTB_MSG_MIN_LEN; + struct xe_gt *gt = ct_to_gt(ct); + struct xe_device *xe = gt_to_xe(gt); + u32 *hxg = msg_to_hxg(msg); + u32 hxg_len = msg_len_to_hxg_len(len); u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); - u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]); + u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); struct g2h_fence *g2h_fence; lockdep_assert_held(&ct->lock); + /* + * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. + * Those messages should never fail, so if we do get an error back it + * means we're likely doing an illegal operation and the GuC is + * rejecting it. We have no way to inform the code that submitted the + * H2G that the message was rejected, so we need to escalate the + * failure to trigger a reset. + */ + if (fence & CT_SEQNO_UNTRACKED) { + if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) + xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", + fence, + FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), + FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); + else + xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", + type, fence); + + return -EPROTO; + } + g2h_fence = xa_erase(&ct->fence_lookup, fence); if (unlikely(!g2h_fence)) { /* Don't tear down channel, as send could've timed out */ - drm_warn(&xe->drm, "G2H fence (%u) not found!\n", fence); + xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); return 0; } @@ -806,18 +945,16 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { g2h_fence->fail = true; - g2h_fence->error = - FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[1]); - g2h_fence->hint = - FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[1]); + g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); + g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { g2h_fence->retry = true; - g2h_fence->reason = - FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, msg[1]); + g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); } else if (g2h_fence->response_buffer) { - g2h_fence->response_len = response_len; - memcpy(g2h_fence->response_buffer, msg + GUC_CTB_MSG_MIN_LEN, - response_len * sizeof(u32)); + g2h_fence->response_len = hxg_len; + memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); + } else { + g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); } g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); @@ -833,14 +970,13 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) { struct xe_device *xe = ct_to_xe(ct); - u32 hxg, origin, type; + u32 *hxg = msg_to_hxg(msg); + u32 origin, type; int ret; lockdep_assert_held(&ct->lock); - hxg = msg[1]; - - origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg); + origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { drm_err(&xe->drm, "G2H channel broken on read, origin=%d, reset required\n", @@ -850,7 +986,7 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) return -EPROTO; } - type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg); + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); switch (type) { case GUC_HXG_TYPE_EVENT: ret = parse_g2h_event(ct, msg, len); @@ -876,14 +1012,19 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) { struct xe_device *xe = ct_to_xe(ct); struct xe_guc *guc = ct_to_guc(ct); - u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]); - u32 *payload = msg + GUC_CTB_HXG_MSG_MIN_LEN; - u32 adj_len = len - GUC_CTB_HXG_MSG_MIN_LEN; + u32 hxg_len = msg_len_to_hxg_len(len); + u32 *hxg = msg_to_hxg(msg); + u32 action, adj_len; + u32 *payload; int ret = 0; - if (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]) != GUC_HXG_TYPE_EVENT) + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) return 0; + action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); + payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; + adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; + switch (action) { case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: ret = xe_guc_sched_done_handler(guc, payload, adj_len); @@ -920,6 +1061,12 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) ret = xe_guc_access_counter_notify_handler(guc, payload, adj_len); break; + case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: + ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); + break; + case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: + ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); + break; default: drm_err(&xe->drm, "unexpected action 0x%04x\n", action); } @@ -938,15 +1085,22 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) u32 tail, head, len; s32 avail; u32 action; + u32 *hxg; + xe_assert(xe, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED); lockdep_assert_held(&ct->fast_lock); - if (!ct->enabled) + if (ct->state == XE_GUC_CT_STATE_DISABLED) return -ENODEV; + if (ct->state == XE_GUC_CT_STATE_STOPPED) + return -ECANCELED; + if (g2h->info.broken) return -EPIPE; + xe_assert(xe, xe_guc_ct_enabled(ct)); + /* Calculate DW available to read */ tail = desc_read(xe, g2h, tail); avail = tail - g2h->info.head; @@ -988,10 +1142,11 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) avail * sizeof(u32)); } - action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]); + hxg = msg_to_hxg(msg); + action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); if (fast_path) { - if (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]) != GUC_HXG_TYPE_EVENT) + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) return 0; switch (action) { @@ -1017,9 +1172,11 @@ static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) { struct xe_device *xe = ct_to_xe(ct); struct xe_guc *guc = ct_to_guc(ct); - u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]); - u32 *payload = msg + GUC_CTB_HXG_MSG_MIN_LEN; - u32 adj_len = len - GUC_CTB_HXG_MSG_MIN_LEN; + u32 hxg_len = msg_len_to_hxg_len(len); + u32 *hxg = msg_to_hxg(msg); + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); + u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; + u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; int ret = 0; switch (action) { @@ -1245,7 +1402,7 @@ struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct, return NULL; } - if (ct->enabled) { + if (xe_guc_ct_enabled(ct)) { snapshot->ct_enabled = true; snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, @@ -1271,7 +1428,7 @@ void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, return; if (snapshot->ct_enabled) { - drm_puts(p, "\nH2G CTB (all sizes in DW):\n"); + drm_puts(p, "H2G CTB (all sizes in DW):\n"); guc_ctb_snapshot_print(&snapshot->h2g, p); drm_puts(p, "\nG2H CTB (all sizes in DW):\n"); @@ -1280,7 +1437,7 @@ void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, drm_printf(p, "\tg2h outstanding: %d\n", snapshot->g2h_outstanding); } else { - drm_puts(p, "\nCT disabled\n"); + drm_puts(p, "CT disabled\n"); } } |