diff options
Diffstat (limited to 'drivers/gpu/drm/i915/vlv_suspend.c')
| -rw-r--r-- | drivers/gpu/drm/i915/vlv_suspend.c | 492 |
1 files changed, 492 insertions, 0 deletions
diff --git a/drivers/gpu/drm/i915/vlv_suspend.c b/drivers/gpu/drm/i915/vlv_suspend.c new file mode 100644 index 000000000..664fde244 --- /dev/null +++ b/drivers/gpu/drm/i915/vlv_suspend.c @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <linux/string_helpers.h> +#include <linux/kernel.h> + +#include <drm/drm_print.h> + +#include "i915_drv.h" +#include "i915_reg.h" +#include "i915_trace.h" +#include "i915_utils.h" +#include "intel_pm.h" +#include "vlv_suspend.h" + +#include "gt/intel_gt_regs.h" + +struct vlv_s0ix_state { + /* GAM */ + u32 wr_watermark; + u32 gfx_prio_ctrl; + u32 arb_mode; + u32 gfx_pend_tlb0; + u32 gfx_pend_tlb1; + u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; + u32 media_max_req_count; + u32 gfx_max_req_count; + u32 render_hwsp; + u32 ecochk; + u32 bsd_hwsp; + u32 blt_hwsp; + u32 tlb_rd_addr; + + /* MBC */ + u32 g3dctl; + u32 gsckgctl; + u32 mbctl; + + /* GCP */ + u32 ucgctl1; + u32 ucgctl3; + u32 rcgctl1; + u32 rcgctl2; + u32 rstctl; + u32 misccpctl; + + /* GPM */ + u32 gfxpause; + u32 rpdeuhwtc; + u32 rpdeuc; + u32 ecobus; + u32 pwrdwnupctl; + u32 rp_down_timeout; + u32 rp_deucsw; + u32 rcubmabdtmr; + u32 rcedata; + u32 spare2gh; + + /* Display 1 CZ domain */ + u32 gt_imr; + u32 gt_ier; + u32 pm_imr; + u32 pm_ier; + u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; + + /* GT SA CZ domain */ + u32 tilectl; + u32 gt_fifoctl; + u32 gtlc_wake_ctrl; + u32 gtlc_survive; + u32 pmwgicz; + + /* Display 2 CZ domain */ + u32 gu_ctl0; + u32 gu_ctl1; + u32 pcbr; + u32 clock_gate_dis2; +}; + +/* + * Save all Gunit registers that may be lost after a D3 and a subsequent + * S0i[R123] transition. The list of registers needing a save/restore is + * defined in the VLV2_S0IXRegs document. This documents marks all Gunit + * registers in the following way: + * - Driver: saved/restored by the driver + * - Punit : saved/restored by the Punit firmware + * - No, w/o marking: no need to save/restore, since the register is R/O or + * used internally by the HW in a way that doesn't depend + * keeping the content across a suspend/resume. + * - Debug : used for debugging + * + * We save/restore all registers marked with 'Driver', with the following + * exceptions: + * - Registers out of use, including also registers marked with 'Debug'. + * These have no effect on the driver's operation, so we don't save/restore + * them to reduce the overhead. + * - Registers that are fully setup by an initialization function called from + * the resume path. For example many clock gating and RPS/RC6 registers. + * - Registers that provide the right functionality with their reset defaults. + * + * TODO: Except for registers that based on the above 3 criteria can be safely + * ignored, we save/restore all others, practically treating the HW context as + * a black-box for the driver. Further investigation is needed to reduce the + * saved/restored registers even further, by following the same 3 criteria. + */ +static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915) +{ + struct vlv_s0ix_state *s = i915->vlv_s0ix_state; + struct intel_uncore *uncore = &i915->uncore; + int i; + + if (!s) + return; + + /* GAM 0x4000-0x4770 */ + s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK); + s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL); + s->arb_mode = intel_uncore_read(uncore, ARB_MODE); + s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0); + s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i)); + + s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT); + s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT); + + s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7); + s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK); + s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7); + s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7); + + s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR); + + /* MBC 0x9024-0x91D0, 0x8500 */ + s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL); + s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL); + s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1); + s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3); + s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1); + s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2); + s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL); + s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE); + s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC); + s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC); + s->ecobus = intel_uncore_read(uncore, ECOBUS); + s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL); + s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT); + s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW); + s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR); + s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA); + s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + s->gt_imr = intel_uncore_read(uncore, GTIMR); + s->gt_ier = intel_uncore_read(uncore, GTIER); + s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); + s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i)); + + /* GT SA CZ domain, 0x100000-0x138124 */ + s->tilectl = intel_uncore_read(uncore, TILECTL); + s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL); + s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL); + s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG); + s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0); + s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1); + s->pcbr = intel_uncore_read(uncore, VLV_PCBR); + s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2); + + /* + * Not saving any of: + * DFT, 0x9800-0x9EC0 + * SARB, 0xB000-0xB1FC + * GAC, 0x5208-0x524C, 0x14000-0x14C000 + * PCI CFG + */ +} + +static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915) +{ + struct vlv_s0ix_state *s = i915->vlv_s0ix_state; + struct intel_uncore *uncore = &i915->uncore; + u32 val; + int i; + + if (!s) + return; + + /* GAM 0x4000-0x4770 */ + intel_uncore_write(uncore, GEN7_WR_WATERMARK, s->wr_watermark); + intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); + intel_uncore_write(uncore, ARB_MODE, s->arb_mode | (0xffff << 16)); + intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); + intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), s->lra_limits[i]); + + intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); + intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); + + intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, s->render_hwsp); + intel_uncore_write(uncore, GAM_ECOCHK, s->ecochk); + intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, s->bsd_hwsp); + intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, s->blt_hwsp); + + intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, s->tlb_rd_addr); + + /* MBC 0x9024-0x91D0, 0x8500 */ + intel_uncore_write(uncore, VLV_G3DCTL, s->g3dctl); + intel_uncore_write(uncore, VLV_GSCKGCTL, s->gsckgctl); + intel_uncore_write(uncore, GEN6_MBCTL, s->mbctl); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + intel_uncore_write(uncore, GEN6_UCGCTL1, s->ucgctl1); + intel_uncore_write(uncore, GEN6_UCGCTL3, s->ucgctl3); + intel_uncore_write(uncore, GEN6_RCGCTL1, s->rcgctl1); + intel_uncore_write(uncore, GEN6_RCGCTL2, s->rcgctl2); + intel_uncore_write(uncore, GEN6_RSTCTL, s->rstctl); + intel_uncore_write(uncore, GEN7_MISCCPCTL, s->misccpctl); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + intel_uncore_write(uncore, GEN6_GFXPAUSE, s->gfxpause); + intel_uncore_write(uncore, GEN6_RPDEUHWTC, s->rpdeuhwtc); + intel_uncore_write(uncore, GEN6_RPDEUC, s->rpdeuc); + intel_uncore_write(uncore, ECOBUS, s->ecobus); + intel_uncore_write(uncore, VLV_PWRDWNUPCTL, s->pwrdwnupctl); + intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, s->rp_down_timeout); + intel_uncore_write(uncore, GEN6_RPDEUCSW, s->rp_deucsw); + intel_uncore_write(uncore, GEN6_RCUBMABDTMR, s->rcubmabdtmr); + intel_uncore_write(uncore, VLV_RCEDATA, s->rcedata); + intel_uncore_write(uncore, VLV_SPAREG2H, s->spare2gh); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + intel_uncore_write(uncore, GTIMR, s->gt_imr); + intel_uncore_write(uncore, GTIER, s->gt_ier); + intel_uncore_write(uncore, GEN6_PMIMR, s->pm_imr); + intel_uncore_write(uncore, GEN6_PMIER, s->pm_ier); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), s->gt_scratch[i]); + + /* GT SA CZ domain, 0x100000-0x138124 */ + intel_uncore_write(uncore, TILECTL, s->tilectl); + intel_uncore_write(uncore, GTFIFOCTL, s->gt_fifoctl); + /* + * Preserve the GT allow wake and GFX force clock bit, they are not + * be restored, as they are used to control the s0ix suspend/resume + * sequence by the caller. + */ + val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL); + val &= VLV_GTLC_ALLOWWAKEREQ; + val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; + intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val); + + val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG); + val &= VLV_GFX_CLK_FORCE_ON_BIT; + val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; + intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val); + + intel_uncore_write(uncore, VLV_PMWGICZ, s->pmwgicz); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + intel_uncore_write(uncore, VLV_GU_CTL0, s->gu_ctl0); + intel_uncore_write(uncore, VLV_GU_CTL1, s->gu_ctl1); + intel_uncore_write(uncore, VLV_PCBR, s->pcbr); + intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); +} + +static int vlv_wait_for_pw_status(struct drm_i915_private *i915, + u32 mask, u32 val) +{ + i915_reg_t reg = VLV_GTLC_PW_STATUS; + u32 reg_value; + int ret; + + /* The HW does not like us polling for PW_STATUS frequently, so + * use the sleeping loop rather than risk the busy spin within + * intel_wait_for_register(). + * + * Transitioning between RC6 states should be at most 2ms (see + * valleyview_enable_rps) so use a 3ms timeout. + */ + ret = wait_for(((reg_value = + intel_uncore_read_notrace(&i915->uncore, reg)) & mask) + == val, 3); + + /* just trace the final value */ + trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); + + return ret; +} + +static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on) +{ + struct intel_uncore *uncore = &i915->uncore; + u32 val; + int err; + + val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG); + val &= ~VLV_GFX_CLK_FORCE_ON_BIT; + if (force_on) + val |= VLV_GFX_CLK_FORCE_ON_BIT; + intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val); + + if (!force_on) + return 0; + + err = intel_wait_for_register(uncore, + VLV_GTLC_SURVIVABILITY_REG, + VLV_GFX_CLK_STATUS_BIT, + VLV_GFX_CLK_STATUS_BIT, + 20); + if (err) + drm_err(&i915->drm, + "timeout waiting for GFX clock force-on (%08x)\n", + intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG)); + + return err; +} + +static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow) +{ + struct intel_uncore *uncore = &i915->uncore; + u32 mask; + u32 val; + int err; + + val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL); + val &= ~VLV_GTLC_ALLOWWAKEREQ; + if (allow) + val |= VLV_GTLC_ALLOWWAKEREQ; + intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val); + intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL); + + mask = VLV_GTLC_ALLOWWAKEACK; + val = allow ? mask : 0; + + err = vlv_wait_for_pw_status(i915, mask, val); + if (err) + drm_err(&i915->drm, "timeout disabling GT waking\n"); + + return err; +} + +static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, + bool wait_for_on) +{ + u32 mask; + u32 val; + + mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; + val = wait_for_on ? mask : 0; + + /* + * RC6 transitioning can be delayed up to 2 msec (see + * valleyview_enable_rps), use 3 msec for safety. + * + * This can fail to turn off the rc6 if the GPU is stuck after a failed + * reset and we are trying to force the machine to sleep. + */ + if (vlv_wait_for_pw_status(dev_priv, mask, val)) + drm_dbg(&dev_priv->drm, + "timeout waiting for GT wells to go %s\n", + str_on_off(wait_for_on)); +} + +static void vlv_check_no_gt_access(struct drm_i915_private *i915) +{ + struct intel_uncore *uncore = &i915->uncore; + + if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) + return; + + drm_dbg(&i915->drm, "GT register access while GT waking disabled\n"); + intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); +} + +int vlv_suspend_complete(struct drm_i915_private *dev_priv) +{ + u32 mask; + int err; + + if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) + return 0; + + /* + * Bspec defines the following GT well on flags as debug only, so + * don't treat them as hard failures. + */ + vlv_wait_for_gt_wells(dev_priv, false); + + mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; + drm_WARN_ON(&dev_priv->drm, + (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask); + + vlv_check_no_gt_access(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, true); + if (err) + goto err1; + + err = vlv_allow_gt_wake(dev_priv, false); + if (err) + goto err2; + + vlv_save_gunit_s0ix_state(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, false); + if (err) + goto err2; + + return 0; + +err2: + /* For safety always re-enable waking and disable gfx clock forcing */ + vlv_allow_gt_wake(dev_priv, true); +err1: + vlv_force_gfx_clock(dev_priv, false); + + return err; +} + +int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume) +{ + int err; + int ret; + + if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) + return 0; + + /* + * If any of the steps fail just try to continue, that's the best we + * can do at this point. Return the first error code (which will also + * leave RPM permanently disabled). + */ + ret = vlv_force_gfx_clock(dev_priv, true); + + vlv_restore_gunit_s0ix_state(dev_priv); + + err = vlv_allow_gt_wake(dev_priv, true); + if (!ret) + ret = err; + + err = vlv_force_gfx_clock(dev_priv, false); + if (!ret) + ret = err; + + vlv_check_no_gt_access(dev_priv); + + if (rpm_resume) + intel_init_clock_gating(dev_priv); + + return ret; +} + +int vlv_suspend_init(struct drm_i915_private *i915) +{ + if (!IS_VALLEYVIEW(i915)) + return 0; + + /* we write all the values in the struct, so no need to zero it out */ + i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state), + GFP_KERNEL); + if (!i915->vlv_s0ix_state) + return -ENOMEM; + + return 0; +} + +void vlv_suspend_cleanup(struct drm_i915_private *i915) +{ + if (!i915->vlv_s0ix_state) + return; + + kfree(i915->vlv_s0ix_state); + i915->vlv_s0ix_state = NULL; +} |