summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/nouveau/nvkm/engine/fifo
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/nouveau/nvkm/engine/fifo')
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild2
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c11
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c55
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h3
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c8
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c7
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c665
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h6
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c4
-rw-r--r--drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c9
12 files changed, 754 insertions, 23 deletions
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
index 5a074b9970..aff92848ab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
@@ -26,5 +26,7 @@ nvkm-y += nvkm/engine/fifo/tu102.o
nvkm-y += nvkm/engine/fifo/ga100.o
nvkm-y += nvkm/engine/fifo/ga102.o
+nvkm-y += nvkm/engine/fifo/r535.o
+
nvkm-y += nvkm/engine/fifo/ucgrp.o
nvkm-y += nvkm/engine/fifo/uchan.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index 5db37247dc..22443fe4a3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -210,6 +210,8 @@ nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data)
CASE(SEC2 );
CASE(NVDEC );
CASE(NVENC );
+ CASE(NVJPG );
+ CASE(OFA );
default:
WARN_ON(1);
break;
@@ -347,8 +349,14 @@ nvkm_fifo_dtor(struct nvkm_engine *engine)
nvkm_chid_unref(&fifo->cgid);
nvkm_chid_unref(&fifo->chid);
+ mutex_destroy(&fifo->userd.mutex);
+
nvkm_event_fini(&fifo->nonstall.event);
mutex_destroy(&fifo->mutex);
+
+ if (fifo->func->dtor)
+ fifo->func->dtor(fifo);
+
return fifo;
}
@@ -383,5 +391,8 @@ nvkm_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device,
spin_lock_init(&fifo->lock);
mutex_init(&fifo->mutex);
+ INIT_LIST_HEAD(&fifo->userd.list);
+ mutex_init(&fifo->userd.mutex);
+
return nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c
index ea53fb3d5d..814db9daa1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c
@@ -156,6 +156,9 @@ nvkm_cgrp_vctx_get(struct nvkm_cgrp *cgrp, struct nvkm_engn *engn, struct nvkm_c
atomic_inc(&vctx->vmm->engref[engn->engine->subdev.type]);
/* Allocate the HW structures. */
+ if (engn->func->ctor2) {
+ ret = engn->func->ctor2(engn, vctx, chan);
+ } else
if (engn->func->bind) {
ret = nvkm_object_bind(vctx->ectx->object, NULL, 0, &vctx->inst);
if (ret == 0 && engn->func->ctor)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
index b7c9d6115b..87a62d4ff4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
@@ -275,13 +275,17 @@ nvkm_chan_del(struct nvkm_chan **pchan)
nvkm_gpuobj_del(&chan->cache);
nvkm_gpuobj_del(&chan->ramfc);
- nvkm_memory_unref(&chan->userd.mem);
-
if (chan->cgrp) {
- nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock);
+ if (!chan->func->id_put)
+ nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock);
+ else
+ chan->func->id_put(chan);
+
nvkm_cgrp_unref(&chan->cgrp);
}
+ nvkm_memory_unref(&chan->userd.mem);
+
if (chan->vmm) {
nvkm_vmm_part(chan->vmm, chan->inst->memory);
nvkm_vmm_unref(&chan->vmm);
@@ -438,7 +442,32 @@ nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int ru
}
/* Allocate channel ID. */
- chan->id = nvkm_chid_get(runl->chid, chan);
+ if (!chan->func->id_get) {
+ chan->id = nvkm_chid_get(runl->chid, chan);
+ if (chan->id >= 0) {
+ if (func->userd->bar < 0) {
+ if (ouserd + chan->func->userd->size >=
+ nvkm_memory_size(userd)) {
+ RUNL_DEBUG(runl, "ouserd %llx", ouserd);
+ return -EINVAL;
+ }
+
+ ret = nvkm_memory_kmap(userd, &chan->userd.mem);
+ if (ret) {
+ RUNL_DEBUG(runl, "userd %d", ret);
+ return ret;
+ }
+
+ chan->userd.base = ouserd;
+ } else {
+ chan->userd.mem = nvkm_memory_ref(fifo->userd.mem);
+ chan->userd.base = chan->id * chan->func->userd->size;
+ }
+ }
+ } else {
+ chan->id = chan->func->id_get(chan, userd, ouserd);
+ }
+
if (chan->id < 0) {
RUNL_ERROR(runl, "!chids");
return -ENOSPC;
@@ -448,24 +477,6 @@ nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int ru
cgrp->id = chan->id;
/* Initialise USERD. */
- if (func->userd->bar < 0) {
- if (ouserd + chan->func->userd->size >= nvkm_memory_size(userd)) {
- RUNL_DEBUG(runl, "ouserd %llx", ouserd);
- return -EINVAL;
- }
-
- ret = nvkm_memory_kmap(userd, &chan->userd.mem);
- if (ret) {
- RUNL_DEBUG(runl, "userd %d", ret);
- return ret;
- }
-
- chan->userd.base = ouserd;
- } else {
- chan->userd.mem = nvkm_memory_ref(fifo->userd.mem);
- chan->userd.base = chan->id * chan->func->userd->size;
- }
-
if (chan->func->userd->clear)
chan->func->userd->clear(chan);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h
index 85b94f6991..013682a709 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h
@@ -17,6 +17,9 @@ struct nvkm_cctx {
};
struct nvkm_chan_func {
+ int (*id_get)(struct nvkm_chan *, struct nvkm_memory *userd, u64 ouserd);
+ void (*id_put)(struct nvkm_chan *);
+
const struct nvkm_chan_func_inst {
u32 size;
bool zero;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c
index c56d2a839e..e74493a456 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c
@@ -27,6 +27,7 @@
#include "runq.h"
#include <core/gpuobj.h>
+#include <subdev/gsp.h>
#include <subdev/top.h>
#include <subdev/vfn.h>
@@ -549,6 +550,10 @@ ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo)
struct nvkm_engn *engn = list_first_entry(&runl->engns, typeof(*engn), head);
runl->nonstall.vector = engn->func->nonstall(engn);
+
+ /* if no nonstall vector just keep going */
+ if (runl->nonstall.vector == -1)
+ continue;
if (runl->nonstall.vector < 0) {
RUNL_ERROR(runl, "nonstall %d", runl->nonstall.vector);
return runl->nonstall.vector;
@@ -607,5 +612,8 @@ int
ga100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
struct nvkm_fifo **pfifo)
{
+ if (nvkm_gsp_rm(device->gsp))
+ return r535_fifo_new(&ga100_fifo, device, type, inst, pfifo);
+
return nvkm_fifo_new_(&ga100_fifo, device, type, inst, pfifo);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c
index 2cdf5da339..755235f55b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c
@@ -21,6 +21,8 @@
*/
#include "priv.h"
+#include <subdev/gsp.h>
+
#include <nvif/class.h>
static const struct nvkm_fifo_func
@@ -34,12 +36,15 @@ ga102_fifo = {
.engn = &ga100_engn,
.engn_ce = &ga100_engn_ce,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true },
- .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_B }, &ga100_chan },
+ .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_A }, &ga100_chan },
};
int
ga102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
struct nvkm_fifo **pfifo)
{
+ if (nvkm_gsp_rm(device->gsp))
+ return r535_fifo_new(&ga102_fifo, device, type, inst, pfifo);
+
return nvkm_fifo_new_(&ga102_fifo, device, type, inst, pfifo);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
index 4d448be192..a0f3277605 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
@@ -13,6 +13,8 @@ struct nvkm_runq;
struct nvkm_vctx;
struct nvkm_fifo_func {
+ void (*dtor)(struct nvkm_fifo *);
+
int (*chid_nr)(struct nvkm_fifo *);
int (*chid_ctor)(struct nvkm_fifo *, int nr);
int (*runq_nr)(struct nvkm_fifo *);
@@ -58,6 +60,8 @@ struct nvkm_fifo_func {
} chan;
};
+int r535_fifo_new(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
+ struct nvkm_fifo **);
int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int,
struct nvkm_fifo **);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c
new file mode 100644
index 0000000000..3454c7d295
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c
@@ -0,0 +1,665 @@
+/*
+ * Copyright 2023 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+#include "cgrp.h"
+#include "chan.h"
+#include "chid.h"
+#include "runl.h"
+
+#include <core/gpuobj.h>
+#include <subdev/gsp.h>
+#include <subdev/mmu.h>
+#include <subdev/vfn.h>
+#include <engine/gr.h>
+
+#include <nvhw/drf.h>
+
+#include <nvrm/nvtypes.h>
+#include <nvrm/535.113.01/common/sdk/nvidia/inc/alloc/alloc_channel.h>
+#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080_notification.h>
+#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080ce.h>
+#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fifo.h>
+#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
+#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
+#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrla06f/ctrla06fgpfifo.h>
+#include <nvrm/535.113.01/nvidia/generated/g_kernel_channel_nvoc.h>
+#include <nvrm/535.113.01/nvidia/generated/g_kernel_fifo_nvoc.h>
+#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gpu_engine_type.h>
+
+static u32
+r535_chan_doorbell_handle(struct nvkm_chan *chan)
+{
+ return (chan->cgrp->runl->id << 16) | chan->id;
+}
+
+static void
+r535_chan_stop(struct nvkm_chan *chan)
+{
+}
+
+static void
+r535_chan_start(struct nvkm_chan *chan)
+{
+}
+
+static void
+r535_chan_ramfc_clear(struct nvkm_chan *chan)
+{
+ struct nvkm_fifo *fifo = chan->cgrp->runl->fifo;
+
+ nvkm_gsp_rm_free(&chan->rm.object);
+
+ dma_free_coherent(fifo->engine.subdev.device->dev, fifo->rm.mthdbuf_size,
+ chan->rm.mthdbuf.ptr, chan->rm.mthdbuf.addr);
+
+ nvkm_cgrp_vctx_put(chan->cgrp, &chan->rm.grctx);
+}
+
+#define CHID_PER_USERD 8
+
+static int
+r535_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv)
+{
+ struct nvkm_fifo *fifo = chan->cgrp->runl->fifo;
+ struct nvkm_engn *engn;
+ struct nvkm_device *device = fifo->engine.subdev.device;
+ NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS *args;
+ const int userd_p = chan->id / CHID_PER_USERD;
+ const int userd_i = chan->id % CHID_PER_USERD;
+ u32 eT = ~0;
+ int ret;
+
+ if (unlikely(device->gr && !device->gr->engine.subdev.oneinit)) {
+ ret = nvkm_subdev_oneinit(&device->gr->engine.subdev);
+ if (ret)
+ return ret;
+ }
+
+ nvkm_runl_foreach_engn(engn, chan->cgrp->runl) {
+ eT = engn->id;
+ break;
+ }
+
+ if (WARN_ON(eT == ~0))
+ return -EINVAL;
+
+ chan->rm.mthdbuf.ptr = dma_alloc_coherent(fifo->engine.subdev.device->dev,
+ fifo->rm.mthdbuf_size,
+ &chan->rm.mthdbuf.addr, GFP_KERNEL);
+ if (!chan->rm.mthdbuf.ptr)
+ return -ENOMEM;
+
+ args = nvkm_gsp_rm_alloc_get(&chan->vmm->rm.device.object, 0xf1f00000 | chan->id,
+ fifo->func->chan.user.oclass, sizeof(*args),
+ &chan->rm.object);
+ if (WARN_ON(IS_ERR(args)))
+ return PTR_ERR(args);
+
+ args->gpFifoOffset = offset;
+ args->gpFifoEntries = length / 8;
+
+ args->flags = NVDEF(NVOS04, FLAGS, CHANNEL_TYPE, PHYSICAL);
+ args->flags |= NVDEF(NVOS04, FLAGS, VPR, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_MAP_REFCOUNTING, FALSE);
+ args->flags |= NVVAL(NVOS04, FLAGS, GROUP_CHANNEL_RUNQUEUE, chan->runq);
+ if (!priv)
+ args->flags |= NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, FALSE);
+ else
+ args->flags |= NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, TRUE);
+ args->flags |= NVDEF(NVOS04, FLAGS, DELAY_CHANNEL_SCHEDULING, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_DENY_PHYSICAL_MODE_CE, FALSE);
+
+ args->flags |= NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_VALUE, userd_i);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_FIXED, FALSE);
+ args->flags |= NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_VALUE, userd_p);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_FIXED, TRUE);
+
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_DENY_AUTH_LEVEL_PRIV, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_SCRUBBER, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_CLIENT_MAP_FIFO, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, SET_EVICT_LAST_CE_PREFETCH_CHANNEL, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_VGPU_PLUGIN_CONTEXT, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_PBDMA_ACQUIRE_TIMEOUT, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, GROUP_CHANNEL_THREAD, DEFAULT);
+ args->flags |= NVDEF(NVOS04, FLAGS, MAP_CHANNEL, FALSE);
+ args->flags |= NVDEF(NVOS04, FLAGS, SKIP_CTXBUFFER_ALLOC, FALSE);
+
+ args->hVASpace = chan->vmm->rm.object.handle;
+ args->engineType = eT;
+
+ args->instanceMem.base = chan->inst->addr;
+ args->instanceMem.size = chan->inst->size;
+ args->instanceMem.addressSpace = 2;
+ args->instanceMem.cacheAttrib = 1;
+
+ args->userdMem.base = nvkm_memory_addr(chan->userd.mem) + chan->userd.base;
+ args->userdMem.size = fifo->func->chan.func->userd->size;
+ args->userdMem.addressSpace = 2;
+ args->userdMem.cacheAttrib = 1;
+
+ args->ramfcMem.base = chan->inst->addr + 0;
+ args->ramfcMem.size = 0x200;
+ args->ramfcMem.addressSpace = 2;
+ args->ramfcMem.cacheAttrib = 1;
+
+ args->mthdbufMem.base = chan->rm.mthdbuf.addr;
+ args->mthdbufMem.size = fifo->rm.mthdbuf_size;
+ args->mthdbufMem.addressSpace = 1;
+ args->mthdbufMem.cacheAttrib = 0;
+
+ if (!priv)
+ args->internalFlags = NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, USER);
+ else
+ args->internalFlags = NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, ADMIN);
+ args->internalFlags |= NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ERROR_NOTIFIER_TYPE, NONE);
+ args->internalFlags |= NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ECC_ERROR_NOTIFIER_TYPE, NONE);
+
+ ret = nvkm_gsp_rm_alloc_wr(&chan->rm.object, args);
+ if (ret)
+ return ret;
+
+ if (1) {
+ NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS *ctrl;
+
+ if (1) {
+ NVA06F_CTRL_BIND_PARAMS *ctrl;
+
+ ctrl = nvkm_gsp_rm_ctrl_get(&chan->rm.object,
+ NVA06F_CTRL_CMD_BIND, sizeof(*ctrl));
+ if (WARN_ON(IS_ERR(ctrl)))
+ return PTR_ERR(ctrl);
+
+ ctrl->engineType = eT;
+
+ ret = nvkm_gsp_rm_ctrl_wr(&chan->rm.object, ctrl);
+ if (ret)
+ return ret;
+ }
+
+ ctrl = nvkm_gsp_rm_ctrl_get(&chan->rm.object,
+ NVA06F_CTRL_CMD_GPFIFO_SCHEDULE, sizeof(*ctrl));
+ if (WARN_ON(IS_ERR(ctrl)))
+ return PTR_ERR(ctrl);
+
+ ctrl->bEnable = 1;
+ ret = nvkm_gsp_rm_ctrl_wr(&chan->rm.object, ctrl);
+ }
+
+ return ret;
+}
+
+static const struct nvkm_chan_func_ramfc
+r535_chan_ramfc = {
+ .write = r535_chan_ramfc_write,
+ .clear = r535_chan_ramfc_clear,
+ .devm = 0xfff,
+ .priv = true,
+};
+
+struct r535_chan_userd {
+ struct nvkm_memory *mem;
+ struct nvkm_memory *map;
+ int chid;
+ u32 used;
+
+ struct list_head head;
+} *userd;
+
+static void
+r535_chan_id_put(struct nvkm_chan *chan)
+{
+ struct nvkm_runl *runl = chan->cgrp->runl;
+ struct nvkm_fifo *fifo = runl->fifo;
+ struct r535_chan_userd *userd;
+
+ mutex_lock(&fifo->userd.mutex);
+ list_for_each_entry(userd, &fifo->userd.list, head) {
+ if (userd->map == chan->userd.mem) {
+ u32 chid = chan->userd.base / chan->func->userd->size;
+
+ userd->used &= ~BIT(chid);
+ if (!userd->used) {
+ nvkm_memory_unref(&userd->map);
+ nvkm_memory_unref(&userd->mem);
+ nvkm_chid_put(runl->chid, userd->chid, &chan->cgrp->lock);
+ list_del(&userd->head);
+ kfree(userd);
+ }
+
+ break;
+ }
+ }
+ mutex_unlock(&fifo->userd.mutex);
+
+}
+
+static int
+r535_chan_id_get_locked(struct nvkm_chan *chan, struct nvkm_memory *muserd, u64 ouserd)
+{
+ const u32 userd_size = CHID_PER_USERD * chan->func->userd->size;
+ struct nvkm_runl *runl = chan->cgrp->runl;
+ struct nvkm_fifo *fifo = runl->fifo;
+ struct r535_chan_userd *userd;
+ u32 chid;
+ int ret;
+
+ if (ouserd + chan->func->userd->size >= userd_size ||
+ (ouserd & (chan->func->userd->size - 1))) {
+ RUNL_DEBUG(runl, "ouserd %llx", ouserd);
+ return -EINVAL;
+ }
+
+ chid = div_u64(ouserd, chan->func->userd->size);
+
+ list_for_each_entry(userd, &fifo->userd.list, head) {
+ if (userd->mem == muserd) {
+ if (userd->used & BIT(chid))
+ return -EBUSY;
+ break;
+ }
+ }
+
+ if (&userd->head == &fifo->userd.list) {
+ if (nvkm_memory_size(muserd) < userd_size) {
+ RUNL_DEBUG(runl, "userd too small");
+ return -EINVAL;
+ }
+
+ userd = kzalloc(sizeof(*userd), GFP_KERNEL);
+ if (!userd)
+ return -ENOMEM;
+
+ userd->chid = nvkm_chid_get(runl->chid, chan);
+ if (userd->chid < 0) {
+ ret = userd->chid;
+ kfree(userd);
+ return ret;
+ }
+
+ userd->mem = nvkm_memory_ref(muserd);
+
+ ret = nvkm_memory_kmap(userd->mem, &userd->map);
+ if (ret) {
+ nvkm_chid_put(runl->chid, userd->chid, &chan->cgrp->lock);
+ kfree(userd);
+ return ret;
+ }
+
+
+ list_add(&userd->head, &fifo->userd.list);
+ }
+
+ userd->used |= BIT(chid);
+
+ chan->userd.mem = nvkm_memory_ref(userd->map);
+ chan->userd.base = ouserd;
+
+ return (userd->chid * CHID_PER_USERD) + chid;
+}
+
+static int
+r535_chan_id_get(struct nvkm_chan *chan, struct nvkm_memory *muserd, u64 ouserd)
+{
+ struct nvkm_fifo *fifo = chan->cgrp->runl->fifo;
+ int ret;
+
+ mutex_lock(&fifo->userd.mutex);
+ ret = r535_chan_id_get_locked(chan, muserd, ouserd);
+ mutex_unlock(&fifo->userd.mutex);
+ return ret;
+}
+
+static const struct nvkm_chan_func
+r535_chan = {
+ .id_get = r535_chan_id_get,
+ .id_put = r535_chan_id_put,
+ .inst = &gf100_chan_inst,
+ .userd = &gv100_chan_userd,
+ .ramfc = &r535_chan_ramfc,
+ .start = r535_chan_start,
+ .stop = r535_chan_stop,
+ .doorbell_handle = r535_chan_doorbell_handle,
+};
+
+static const struct nvkm_cgrp_func
+r535_cgrp = {
+};
+
+static int
+r535_engn_nonstall(struct nvkm_engn *engn)
+{
+ struct nvkm_subdev *subdev = &engn->engine->subdev;
+ int ret;
+
+ ret = nvkm_gsp_intr_nonstall(subdev->device->gsp, subdev->type, subdev->inst);
+ WARN_ON(ret == -ENOENT);
+ return ret;
+}
+
+static const struct nvkm_engn_func
+r535_ce = {
+ .nonstall = r535_engn_nonstall,
+};
+
+static int
+r535_gr_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan)
+{
+ /* RM requires GR context buffers to remain mapped until after the
+ * channel has been destroyed (as opposed to after the last gr obj
+ * has been deleted).
+ *
+ * Take an extra ref here, which will be released once the channel
+ * object has been deleted.
+ */
+ refcount_inc(&vctx->refs);
+ chan->rm.grctx = vctx;
+ return 0;
+}
+
+static const struct nvkm_engn_func
+r535_gr = {
+ .nonstall = r535_engn_nonstall,
+ .ctor2 = r535_gr_ctor,
+};
+
+static int
+r535_flcn_bind(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan)
+{
+ struct nvkm_gsp_client *client = &chan->vmm->rm.client;
+ NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS *ctrl;
+
+ ctrl = nvkm_gsp_rm_ctrl_get(&chan->vmm->rm.device.subdevice,
+ NV2080_CTRL_CMD_GPU_PROMOTE_CTX, sizeof(*ctrl));
+ if (IS_ERR(ctrl))
+ return PTR_ERR(ctrl);
+
+ ctrl->hClient = client->object.handle;
+ ctrl->hObject = chan->rm.object.handle;
+ ctrl->hChanClient = client->object.handle;
+ ctrl->virtAddress = vctx->vma->addr;
+ ctrl->size = vctx->inst->size;
+ ctrl->engineType = engn->id;
+ ctrl->ChID = chan->id;
+
+ return nvkm_gsp_rm_ctrl_wr(&chan->vmm->rm.device.subdevice, ctrl);
+}
+
+static int
+r535_flcn_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan)
+{
+ int ret;
+
+ if (WARN_ON(!engn->rm.size))
+ return -EINVAL;
+
+ ret = nvkm_gpuobj_new(engn->engine->subdev.device, engn->rm.size, 0, true, NULL,
+ &vctx->inst);
+ if (ret)
+ return ret;
+
+ ret = nvkm_vmm_get(vctx->vmm, 12, vctx->inst->size, &vctx->vma);
+ if (ret)
+ return ret;
+
+ ret = nvkm_memory_map(vctx->inst, 0, vctx->vmm, vctx->vma, NULL, 0);
+ if (ret)
+ return ret;
+
+ return r535_flcn_bind(engn, vctx, chan);
+}
+
+static const struct nvkm_engn_func
+r535_flcn = {
+ .nonstall = r535_engn_nonstall,
+ .ctor2 = r535_flcn_ctor,
+};
+
+static void
+r535_runl_allow(struct nvkm_runl *runl, u32 engm)
+{
+}
+
+static void
+r535_runl_block(struct nvkm_runl *runl, u32 engm)
+{
+}
+
+static const struct nvkm_runl_func
+r535_runl = {
+ .block = r535_runl_block,
+ .allow = r535_runl_allow,
+};
+
+static int
+r535_fifo_2080_type(enum nvkm_subdev_type type, int inst)
+{
+ switch (type) {
+ case NVKM_ENGINE_GR: return NV2080_ENGINE_TYPE_GR0;
+ case NVKM_ENGINE_CE: return NV2080_ENGINE_TYPE_COPY0 + inst;
+ case NVKM_ENGINE_SEC2: return NV2080_ENGINE_TYPE_SEC2;
+ case NVKM_ENGINE_NVDEC: return NV2080_ENGINE_TYPE_NVDEC0 + inst;
+ case NVKM_ENGINE_NVENC: return NV2080_ENGINE_TYPE_NVENC0 + inst;
+ case NVKM_ENGINE_NVJPG: return NV2080_ENGINE_TYPE_NVJPEG0 + inst;
+ case NVKM_ENGINE_OFA: return NV2080_ENGINE_TYPE_OFA;
+ case NVKM_ENGINE_SW: return NV2080_ENGINE_TYPE_SW;
+ default:
+ break;
+ }
+
+ WARN_ON(1);
+ return -EINVAL;
+}
+
+static int
+r535_fifo_engn_type(RM_ENGINE_TYPE rm, enum nvkm_subdev_type *ptype)
+{
+ switch (rm) {
+ case RM_ENGINE_TYPE_GR0:
+ *ptype = NVKM_ENGINE_GR;
+ return 0;
+ case RM_ENGINE_TYPE_COPY0...RM_ENGINE_TYPE_COPY9:
+ *ptype = NVKM_ENGINE_CE;
+ return rm - RM_ENGINE_TYPE_COPY0;
+ case RM_ENGINE_TYPE_NVDEC0...RM_ENGINE_TYPE_NVDEC7:
+ *ptype = NVKM_ENGINE_NVDEC;
+ return rm - RM_ENGINE_TYPE_NVDEC0;
+ case RM_ENGINE_TYPE_NVENC0...RM_ENGINE_TYPE_NVENC2:
+ *ptype = NVKM_ENGINE_NVENC;
+ return rm - RM_ENGINE_TYPE_NVENC0;
+ case RM_ENGINE_TYPE_SW:
+ *ptype = NVKM_ENGINE_SW;
+ return 0;
+ case RM_ENGINE_TYPE_SEC2:
+ *ptype = NVKM_ENGINE_SEC2;
+ return 0;
+ case RM_ENGINE_TYPE_NVJPEG0...RM_ENGINE_TYPE_NVJPEG7:
+ *ptype = NVKM_ENGINE_NVJPG;
+ return rm - RM_ENGINE_TYPE_NVJPEG0;
+ case RM_ENGINE_TYPE_OFA:
+ *ptype = NVKM_ENGINE_OFA;
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int
+r535_fifo_ectx_size(struct nvkm_fifo *fifo)
+{
+ NV2080_CTRL_INTERNAL_GET_CONSTRUCTED_FALCON_INFO_PARAMS *ctrl;
+ struct nvkm_gsp *gsp = fifo->engine.subdev.device->gsp;
+ struct nvkm_runl *runl;
+ struct nvkm_engn *engn;
+
+ ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
+ NV2080_CTRL_CMD_INTERNAL_GET_CONSTRUCTED_FALCON_INFO,
+ sizeof(*ctrl));
+ if (WARN_ON(IS_ERR(ctrl)))
+ return PTR_ERR(ctrl);
+
+ for (int i = 0; i < ctrl->numConstructedFalcons; i++) {
+ nvkm_runl_foreach(runl, fifo) {
+ nvkm_runl_foreach_engn(engn, runl) {
+ if (engn->rm.desc == ctrl->constructedFalconsTable[i].engDesc) {
+ engn->rm.size =
+ ctrl->constructedFalconsTable[i].ctxBufferSize;
+ break;
+ }
+ }
+ }
+ }
+
+ nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
+ return 0;
+}
+
+static int
+r535_fifo_runl_ctor(struct nvkm_fifo *fifo)
+{
+ struct nvkm_subdev *subdev = &fifo->engine.subdev;
+ struct nvkm_gsp *gsp = subdev->device->gsp;
+ struct nvkm_runl *runl;
+ struct nvkm_engn *engn;
+ u32 cgids = 2048;
+ u32 chids = 2048;
+ int ret;
+ NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS *ctrl;
+
+ if ((ret = nvkm_chid_new(&nvkm_chan_event, subdev, cgids, 0, cgids, &fifo->cgid)) ||
+ (ret = nvkm_chid_new(&nvkm_chan_event, subdev, chids, 0, chids, &fifo->chid)))
+ return ret;
+
+ ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
+ NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE, sizeof(*ctrl));
+ if (WARN_ON(IS_ERR(ctrl)))
+ return PTR_ERR(ctrl);
+
+ for (int i = 0; i < ctrl->numEntries; i++) {
+ const u32 addr = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST_PRI_BASE];
+ const u32 id = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST];
+
+ runl = nvkm_runl_get(fifo, id, addr);
+ if (!runl) {
+ runl = nvkm_runl_new(fifo, id, addr, 0);
+ if (WARN_ON(IS_ERR(runl)))
+ continue;
+ }
+ }
+
+ for (int i = 0; i < ctrl->numEntries; i++) {
+ const u32 addr = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST_PRI_BASE];
+ const u32 rmid = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE];
+ const u32 id = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST];
+ enum nvkm_subdev_type type;
+ int inst, nv2080;
+
+ runl = nvkm_runl_get(fifo, id, addr);
+ if (!runl)
+ continue;
+
+ inst = r535_fifo_engn_type(rmid, &type);
+ if (inst < 0) {
+ nvkm_warn(subdev, "RM_ENGINE_TYPE 0x%x\n", rmid);
+ nvkm_runl_del(runl);
+ continue;
+ }
+
+ nv2080 = r535_fifo_2080_type(type, inst);
+ if (nv2080 < 0) {
+ nvkm_runl_del(runl);
+ continue;
+ }
+
+ switch (type) {
+ case NVKM_ENGINE_CE:
+ engn = nvkm_runl_add(runl, nv2080, &r535_ce, type, inst);
+ break;
+ case NVKM_ENGINE_GR:
+ engn = nvkm_runl_add(runl, nv2080, &r535_gr, type, inst);
+ break;
+ case NVKM_ENGINE_NVDEC:
+ case NVKM_ENGINE_NVENC:
+ case NVKM_ENGINE_NVJPG:
+ case NVKM_ENGINE_OFA:
+ engn = nvkm_runl_add(runl, nv2080, &r535_flcn, type, inst);
+ break;
+ case NVKM_ENGINE_SW:
+ continue;
+ default:
+ engn = NULL;
+ break;
+ }
+
+ if (!engn) {
+ nvkm_runl_del(runl);
+ continue;
+ }
+
+ engn->rm.desc = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_ENG_DESC];
+ }
+
+ nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
+
+ {
+ NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS *ctrl;
+
+ ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
+ NV2080_CTRL_CMD_CE_GET_FAULT_METHOD_BUFFER_SIZE,
+ sizeof(*ctrl));
+ if (IS_ERR(ctrl))
+ return PTR_ERR(ctrl);
+
+ fifo->rm.mthdbuf_size = ctrl->size;
+
+ nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
+ }
+
+ return r535_fifo_ectx_size(fifo);
+}
+
+static void
+r535_fifo_dtor(struct nvkm_fifo *fifo)
+{
+ kfree(fifo->func);
+}
+
+int
+r535_fifo_new(const struct nvkm_fifo_func *hw, struct nvkm_device *device,
+ enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo)
+{
+ struct nvkm_fifo_func *rm;
+
+ if (!(rm = kzalloc(sizeof(*rm), GFP_KERNEL)))
+ return -ENOMEM;
+
+ rm->dtor = r535_fifo_dtor;
+ rm->runl_ctor = r535_fifo_runl_ctor;
+ rm->runl = &r535_runl;
+ rm->cgrp = hw->cgrp;
+ rm->cgrp.func = &r535_cgrp;
+ rm->chan = hw->chan;
+ rm->chan.func = &r535_chan;
+ rm->nonstall = &ga100_fifo_nonstall;
+ rm->nonstall_ctor = ga100_fifo_nonstall_ctor;
+
+ return nvkm_fifo_new_(rm, device, type, inst, pfifo);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
index 5421321f8e..19e6772ead 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
@@ -18,6 +18,7 @@ struct nvkm_engn {
bool (*mmu_fault_triggered)(struct nvkm_engn *);
int (*ctor)(struct nvkm_engn *, struct nvkm_vctx *);
void (*bind)(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *);
+ int (*ctor2)(struct nvkm_engn *, struct nvkm_vctx *, struct nvkm_chan *);
int (*ramht_add)(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *);
void (*ramht_del)(struct nvkm_chan *, int hash);
} *func;
@@ -28,6 +29,11 @@ struct nvkm_engn {
int fault;
+ struct {
+ u32 desc;
+ u32 size;
+ } rm;
+
struct list_head head;
};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
index ea9e151dbb..1d39a6840a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
@@ -25,6 +25,7 @@
#include "runl.h"
#include <core/memory.h>
+#include <subdev/gsp.h>
#include <subdev/mc.h>
#include <subdev/vfn.h>
@@ -282,5 +283,8 @@ int
tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
struct nvkm_fifo **pfifo)
{
+ if (nvkm_gsp_rm(device->gsp))
+ return r535_fifo_new(&tu102_fifo, device, type, inst, pfifo);
+
return nvkm_fifo_new_(&tu102_fifo, device, type, inst, pfifo);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c
index 04140e0110..9e56bcc166 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c
@@ -317,6 +317,15 @@ nvkm_uchan = {
.uevent = nvkm_uchan_uevent,
};
+struct nvkm_chan *
+nvkm_uchan_chan(struct nvkm_object *object)
+{
+ if (WARN_ON(object->func != &nvkm_uchan))
+ return NULL;
+
+ return nvkm_uchan(object)->chan;
+}
+
int
nvkm_uchan_new(struct nvkm_fifo *fifo, struct nvkm_cgrp *cgrp, const struct nvkm_oclass *oclass,
void *argv, u32 argc, struct nvkm_object **pobject)