diff options
Diffstat (limited to 'drivers/gpu/drm/nouveau/nvkm/engine/fifo')
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c | 11 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 55 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c | 665 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 9 |
12 files changed, 754 insertions, 23 deletions
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 5a074b997..aff92848a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -26,5 +26,7 @@ nvkm-y += nvkm/engine/fifo/tu102.o nvkm-y += nvkm/engine/fifo/ga100.o nvkm-y += nvkm/engine/fifo/ga102.o +nvkm-y += nvkm/engine/fifo/r535.o + nvkm-y += nvkm/engine/fifo/ucgrp.o nvkm-y += nvkm/engine/fifo/uchan.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 5db37247d..22443fe4a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -210,6 +210,8 @@ nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) CASE(SEC2 ); CASE(NVDEC ); CASE(NVENC ); + CASE(NVJPG ); + CASE(OFA ); default: WARN_ON(1); break; @@ -347,8 +349,14 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) nvkm_chid_unref(&fifo->cgid); nvkm_chid_unref(&fifo->chid); + mutex_destroy(&fifo->userd.mutex); + nvkm_event_fini(&fifo->nonstall.event); mutex_destroy(&fifo->mutex); + + if (fifo->func->dtor) + fifo->func->dtor(fifo); + return fifo; } @@ -383,5 +391,8 @@ nvkm_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, spin_lock_init(&fifo->lock); mutex_init(&fifo->mutex); + INIT_LIST_HEAD(&fifo->userd.list); + mutex_init(&fifo->userd.mutex); + return nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c index ea53fb3d5..814db9daa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c @@ -156,6 +156,9 @@ nvkm_cgrp_vctx_get(struct nvkm_cgrp *cgrp, struct nvkm_engn *engn, struct nvkm_c atomic_inc(&vctx->vmm->engref[engn->engine->subdev.type]); /* Allocate the HW structures. */ + if (engn->func->ctor2) { + ret = engn->func->ctor2(engn, vctx, chan); + } else if (engn->func->bind) { ret = nvkm_object_bind(vctx->ectx->object, NULL, 0, &vctx->inst); if (ret == 0 && engn->func->ctor) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index b7c9d6115..87a62d4ff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -275,13 +275,17 @@ nvkm_chan_del(struct nvkm_chan **pchan) nvkm_gpuobj_del(&chan->cache); nvkm_gpuobj_del(&chan->ramfc); - nvkm_memory_unref(&chan->userd.mem); - if (chan->cgrp) { - nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock); + if (!chan->func->id_put) + nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock); + else + chan->func->id_put(chan); + nvkm_cgrp_unref(&chan->cgrp); } + nvkm_memory_unref(&chan->userd.mem); + if (chan->vmm) { nvkm_vmm_part(chan->vmm, chan->inst->memory); nvkm_vmm_unref(&chan->vmm); @@ -438,7 +442,32 @@ nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int ru } /* Allocate channel ID. */ - chan->id = nvkm_chid_get(runl->chid, chan); + if (!chan->func->id_get) { + chan->id = nvkm_chid_get(runl->chid, chan); + if (chan->id >= 0) { + if (func->userd->bar < 0) { + if (ouserd + chan->func->userd->size >= + nvkm_memory_size(userd)) { + RUNL_DEBUG(runl, "ouserd %llx", ouserd); + return -EINVAL; + } + + ret = nvkm_memory_kmap(userd, &chan->userd.mem); + if (ret) { + RUNL_DEBUG(runl, "userd %d", ret); + return ret; + } + + chan->userd.base = ouserd; + } else { + chan->userd.mem = nvkm_memory_ref(fifo->userd.mem); + chan->userd.base = chan->id * chan->func->userd->size; + } + } + } else { + chan->id = chan->func->id_get(chan, userd, ouserd); + } + if (chan->id < 0) { RUNL_ERROR(runl, "!chids"); return -ENOSPC; @@ -448,24 +477,6 @@ nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int ru cgrp->id = chan->id; /* Initialise USERD. */ - if (func->userd->bar < 0) { - if (ouserd + chan->func->userd->size >= nvkm_memory_size(userd)) { - RUNL_DEBUG(runl, "ouserd %llx", ouserd); - return -EINVAL; - } - - ret = nvkm_memory_kmap(userd, &chan->userd.mem); - if (ret) { - RUNL_DEBUG(runl, "userd %d", ret); - return ret; - } - - chan->userd.base = ouserd; - } else { - chan->userd.mem = nvkm_memory_ref(fifo->userd.mem); - chan->userd.base = chan->id * chan->func->userd->size; - } - if (chan->func->userd->clear) chan->func->userd->clear(chan); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 85b94f699..013682a70 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -17,6 +17,9 @@ struct nvkm_cctx { }; struct nvkm_chan_func { + int (*id_get)(struct nvkm_chan *, struct nvkm_memory *userd, u64 ouserd); + void (*id_put)(struct nvkm_chan *); + const struct nvkm_chan_func_inst { u32 size; bool zero; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c index c56d2a839..e74493a45 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c @@ -27,6 +27,7 @@ #include "runq.h" #include <core/gpuobj.h> +#include <subdev/gsp.h> #include <subdev/top.h> #include <subdev/vfn.h> @@ -549,6 +550,10 @@ ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo) struct nvkm_engn *engn = list_first_entry(&runl->engns, typeof(*engn), head); runl->nonstall.vector = engn->func->nonstall(engn); + + /* if no nonstall vector just keep going */ + if (runl->nonstall.vector == -1) + continue; if (runl->nonstall.vector < 0) { RUNL_ERROR(runl, "nonstall %d", runl->nonstall.vector); return runl->nonstall.vector; @@ -607,5 +612,8 @@ int ga100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { + if (nvkm_gsp_rm(device->gsp)) + return r535_fifo_new(&ga100_fifo, device, type, inst, pfifo); + return nvkm_fifo_new_(&ga100_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c index 2cdf5da33..755235f55 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c @@ -21,6 +21,8 @@ */ #include "priv.h" +#include <subdev/gsp.h> + #include <nvif/class.h> static const struct nvkm_fifo_func @@ -34,12 +36,15 @@ ga102_fifo = { .engn = &ga100_engn, .engn_ce = &ga100_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true }, - .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_B }, &ga100_chan }, + .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_A }, &ga100_chan }, }; int ga102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { + if (nvkm_gsp_rm(device->gsp)) + return r535_fifo_new(&ga102_fifo, device, type, inst, pfifo); + return nvkm_fifo_new_(&ga102_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 4d448be19..a0f327760 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -13,6 +13,8 @@ struct nvkm_runq; struct nvkm_vctx; struct nvkm_fifo_func { + void (*dtor)(struct nvkm_fifo *); + int (*chid_nr)(struct nvkm_fifo *); int (*chid_ctor)(struct nvkm_fifo *, int nr); int (*runq_nr)(struct nvkm_fifo *); @@ -58,6 +60,8 @@ struct nvkm_fifo_func { } chan; }; +int r535_fifo_new(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, + struct nvkm_fifo **); int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_fifo **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c new file mode 100644 index 000000000..3454c7d29 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c @@ -0,0 +1,665 @@ +/* + * Copyright 2023 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" +#include "cgrp.h" +#include "chan.h" +#include "chid.h" +#include "runl.h" + +#include <core/gpuobj.h> +#include <subdev/gsp.h> +#include <subdev/mmu.h> +#include <subdev/vfn.h> +#include <engine/gr.h> + +#include <nvhw/drf.h> + +#include <nvrm/nvtypes.h> +#include <nvrm/535.113.01/common/sdk/nvidia/inc/alloc/alloc_channel.h> +#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080_notification.h> +#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080ce.h> +#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fifo.h> +#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h> +#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h> +#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrla06f/ctrla06fgpfifo.h> +#include <nvrm/535.113.01/nvidia/generated/g_kernel_channel_nvoc.h> +#include <nvrm/535.113.01/nvidia/generated/g_kernel_fifo_nvoc.h> +#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gpu_engine_type.h> + +static u32 +r535_chan_doorbell_handle(struct nvkm_chan *chan) +{ + return (chan->cgrp->runl->id << 16) | chan->id; +} + +static void +r535_chan_stop(struct nvkm_chan *chan) +{ +} + +static void +r535_chan_start(struct nvkm_chan *chan) +{ +} + +static void +r535_chan_ramfc_clear(struct nvkm_chan *chan) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + + nvkm_gsp_rm_free(&chan->rm.object); + + dma_free_coherent(fifo->engine.subdev.device->dev, fifo->rm.mthdbuf_size, + chan->rm.mthdbuf.ptr, chan->rm.mthdbuf.addr); + + nvkm_cgrp_vctx_put(chan->cgrp, &chan->rm.grctx); +} + +#define CHID_PER_USERD 8 + +static int +r535_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + struct nvkm_engn *engn; + struct nvkm_device *device = fifo->engine.subdev.device; + NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS *args; + const int userd_p = chan->id / CHID_PER_USERD; + const int userd_i = chan->id % CHID_PER_USERD; + u32 eT = ~0; + int ret; + + if (unlikely(device->gr && !device->gr->engine.subdev.oneinit)) { + ret = nvkm_subdev_oneinit(&device->gr->engine.subdev); + if (ret) + return ret; + } + + nvkm_runl_foreach_engn(engn, chan->cgrp->runl) { + eT = engn->id; + break; + } + + if (WARN_ON(eT == ~0)) + return -EINVAL; + + chan->rm.mthdbuf.ptr = dma_alloc_coherent(fifo->engine.subdev.device->dev, + fifo->rm.mthdbuf_size, + &chan->rm.mthdbuf.addr, GFP_KERNEL); + if (!chan->rm.mthdbuf.ptr) + return -ENOMEM; + + args = nvkm_gsp_rm_alloc_get(&chan->vmm->rm.device.object, 0xf1f00000 | chan->id, + fifo->func->chan.user.oclass, sizeof(*args), + &chan->rm.object); + if (WARN_ON(IS_ERR(args))) + return PTR_ERR(args); + + args->gpFifoOffset = offset; + args->gpFifoEntries = length / 8; + + args->flags = NVDEF(NVOS04, FLAGS, CHANNEL_TYPE, PHYSICAL); + args->flags |= NVDEF(NVOS04, FLAGS, VPR, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_MAP_REFCOUNTING, FALSE); + args->flags |= NVVAL(NVOS04, FLAGS, GROUP_CHANNEL_RUNQUEUE, chan->runq); + if (!priv) + args->flags |= NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, FALSE); + else + args->flags |= NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, TRUE); + args->flags |= NVDEF(NVOS04, FLAGS, DELAY_CHANNEL_SCHEDULING, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_DENY_PHYSICAL_MODE_CE, FALSE); + + args->flags |= NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_VALUE, userd_i); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_FIXED, FALSE); + args->flags |= NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_VALUE, userd_p); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_FIXED, TRUE); + + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_DENY_AUTH_LEVEL_PRIV, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_SCRUBBER, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_CLIENT_MAP_FIFO, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, SET_EVICT_LAST_CE_PREFETCH_CHANNEL, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_VGPU_PLUGIN_CONTEXT, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_PBDMA_ACQUIRE_TIMEOUT, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, GROUP_CHANNEL_THREAD, DEFAULT); + args->flags |= NVDEF(NVOS04, FLAGS, MAP_CHANNEL, FALSE); + args->flags |= NVDEF(NVOS04, FLAGS, SKIP_CTXBUFFER_ALLOC, FALSE); + + args->hVASpace = chan->vmm->rm.object.handle; + args->engineType = eT; + + args->instanceMem.base = chan->inst->addr; + args->instanceMem.size = chan->inst->size; + args->instanceMem.addressSpace = 2; + args->instanceMem.cacheAttrib = 1; + + args->userdMem.base = nvkm_memory_addr(chan->userd.mem) + chan->userd.base; + args->userdMem.size = fifo->func->chan.func->userd->size; + args->userdMem.addressSpace = 2; + args->userdMem.cacheAttrib = 1; + + args->ramfcMem.base = chan->inst->addr + 0; + args->ramfcMem.size = 0x200; + args->ramfcMem.addressSpace = 2; + args->ramfcMem.cacheAttrib = 1; + + args->mthdbufMem.base = chan->rm.mthdbuf.addr; + args->mthdbufMem.size = fifo->rm.mthdbuf_size; + args->mthdbufMem.addressSpace = 1; + args->mthdbufMem.cacheAttrib = 0; + + if (!priv) + args->internalFlags = NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, USER); + else + args->internalFlags = NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, ADMIN); + args->internalFlags |= NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ERROR_NOTIFIER_TYPE, NONE); + args->internalFlags |= NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ECC_ERROR_NOTIFIER_TYPE, NONE); + + ret = nvkm_gsp_rm_alloc_wr(&chan->rm.object, args); + if (ret) + return ret; + + if (1) { + NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS *ctrl; + + if (1) { + NVA06F_CTRL_BIND_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_get(&chan->rm.object, + NVA06F_CTRL_CMD_BIND, sizeof(*ctrl)); + if (WARN_ON(IS_ERR(ctrl))) + return PTR_ERR(ctrl); + + ctrl->engineType = eT; + + ret = nvkm_gsp_rm_ctrl_wr(&chan->rm.object, ctrl); + if (ret) + return ret; + } + + ctrl = nvkm_gsp_rm_ctrl_get(&chan->rm.object, + NVA06F_CTRL_CMD_GPFIFO_SCHEDULE, sizeof(*ctrl)); + if (WARN_ON(IS_ERR(ctrl))) + return PTR_ERR(ctrl); + + ctrl->bEnable = 1; + ret = nvkm_gsp_rm_ctrl_wr(&chan->rm.object, ctrl); + } + + return ret; +} + +static const struct nvkm_chan_func_ramfc +r535_chan_ramfc = { + .write = r535_chan_ramfc_write, + .clear = r535_chan_ramfc_clear, + .devm = 0xfff, + .priv = true, +}; + +struct r535_chan_userd { + struct nvkm_memory *mem; + struct nvkm_memory *map; + int chid; + u32 used; + + struct list_head head; +} *userd; + +static void +r535_chan_id_put(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + struct nvkm_fifo *fifo = runl->fifo; + struct r535_chan_userd *userd; + + mutex_lock(&fifo->userd.mutex); + list_for_each_entry(userd, &fifo->userd.list, head) { + if (userd->map == chan->userd.mem) { + u32 chid = chan->userd.base / chan->func->userd->size; + + userd->used &= ~BIT(chid); + if (!userd->used) { + nvkm_memory_unref(&userd->map); + nvkm_memory_unref(&userd->mem); + nvkm_chid_put(runl->chid, userd->chid, &chan->cgrp->lock); + list_del(&userd->head); + kfree(userd); + } + + break; + } + } + mutex_unlock(&fifo->userd.mutex); + +} + +static int +r535_chan_id_get_locked(struct nvkm_chan *chan, struct nvkm_memory *muserd, u64 ouserd) +{ + const u32 userd_size = CHID_PER_USERD * chan->func->userd->size; + struct nvkm_runl *runl = chan->cgrp->runl; + struct nvkm_fifo *fifo = runl->fifo; + struct r535_chan_userd *userd; + u32 chid; + int ret; + + if (ouserd + chan->func->userd->size >= userd_size || + (ouserd & (chan->func->userd->size - 1))) { + RUNL_DEBUG(runl, "ouserd %llx", ouserd); + return -EINVAL; + } + + chid = div_u64(ouserd, chan->func->userd->size); + + list_for_each_entry(userd, &fifo->userd.list, head) { + if (userd->mem == muserd) { + if (userd->used & BIT(chid)) + return -EBUSY; + break; + } + } + + if (&userd->head == &fifo->userd.list) { + if (nvkm_memory_size(muserd) < userd_size) { + RUNL_DEBUG(runl, "userd too small"); + return -EINVAL; + } + + userd = kzalloc(sizeof(*userd), GFP_KERNEL); + if (!userd) + return -ENOMEM; + + userd->chid = nvkm_chid_get(runl->chid, chan); + if (userd->chid < 0) { + ret = userd->chid; + kfree(userd); + return ret; + } + + userd->mem = nvkm_memory_ref(muserd); + + ret = nvkm_memory_kmap(userd->mem, &userd->map); + if (ret) { + nvkm_chid_put(runl->chid, userd->chid, &chan->cgrp->lock); + kfree(userd); + return ret; + } + + + list_add(&userd->head, &fifo->userd.list); + } + + userd->used |= BIT(chid); + + chan->userd.mem = nvkm_memory_ref(userd->map); + chan->userd.base = ouserd; + + return (userd->chid * CHID_PER_USERD) + chid; +} + +static int +r535_chan_id_get(struct nvkm_chan *chan, struct nvkm_memory *muserd, u64 ouserd) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + int ret; + + mutex_lock(&fifo->userd.mutex); + ret = r535_chan_id_get_locked(chan, muserd, ouserd); + mutex_unlock(&fifo->userd.mutex); + return ret; +} + +static const struct nvkm_chan_func +r535_chan = { + .id_get = r535_chan_id_get, + .id_put = r535_chan_id_put, + .inst = &gf100_chan_inst, + .userd = &gv100_chan_userd, + .ramfc = &r535_chan_ramfc, + .start = r535_chan_start, + .stop = r535_chan_stop, + .doorbell_handle = r535_chan_doorbell_handle, +}; + +static const struct nvkm_cgrp_func +r535_cgrp = { +}; + +static int +r535_engn_nonstall(struct nvkm_engn *engn) +{ + struct nvkm_subdev *subdev = &engn->engine->subdev; + int ret; + + ret = nvkm_gsp_intr_nonstall(subdev->device->gsp, subdev->type, subdev->inst); + WARN_ON(ret == -ENOENT); + return ret; +} + +static const struct nvkm_engn_func +r535_ce = { + .nonstall = r535_engn_nonstall, +}; + +static int +r535_gr_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan) +{ + /* RM requires GR context buffers to remain mapped until after the + * channel has been destroyed (as opposed to after the last gr obj + * has been deleted). + * + * Take an extra ref here, which will be released once the channel + * object has been deleted. + */ + refcount_inc(&vctx->refs); + chan->rm.grctx = vctx; + return 0; +} + +static const struct nvkm_engn_func +r535_gr = { + .nonstall = r535_engn_nonstall, + .ctor2 = r535_gr_ctor, +}; + +static int +r535_flcn_bind(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan) +{ + struct nvkm_gsp_client *client = &chan->vmm->rm.client; + NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_get(&chan->vmm->rm.device.subdevice, + NV2080_CTRL_CMD_GPU_PROMOTE_CTX, sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + ctrl->hClient = client->object.handle; + ctrl->hObject = chan->rm.object.handle; + ctrl->hChanClient = client->object.handle; + ctrl->virtAddress = vctx->vma->addr; + ctrl->size = vctx->inst->size; + ctrl->engineType = engn->id; + ctrl->ChID = chan->id; + + return nvkm_gsp_rm_ctrl_wr(&chan->vmm->rm.device.subdevice, ctrl); +} + +static int +r535_flcn_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan) +{ + int ret; + + if (WARN_ON(!engn->rm.size)) + return -EINVAL; + + ret = nvkm_gpuobj_new(engn->engine->subdev.device, engn->rm.size, 0, true, NULL, + &vctx->inst); + if (ret) + return ret; + + ret = nvkm_vmm_get(vctx->vmm, 12, vctx->inst->size, &vctx->vma); + if (ret) + return ret; + + ret = nvkm_memory_map(vctx->inst, 0, vctx->vmm, vctx->vma, NULL, 0); + if (ret) + return ret; + + return r535_flcn_bind(engn, vctx, chan); +} + +static const struct nvkm_engn_func +r535_flcn = { + .nonstall = r535_engn_nonstall, + .ctor2 = r535_flcn_ctor, +}; + +static void +r535_runl_allow(struct nvkm_runl *runl, u32 engm) +{ +} + +static void +r535_runl_block(struct nvkm_runl *runl, u32 engm) +{ +} + +static const struct nvkm_runl_func +r535_runl = { + .block = r535_runl_block, + .allow = r535_runl_allow, +}; + +static int +r535_fifo_2080_type(enum nvkm_subdev_type type, int inst) +{ + switch (type) { + case NVKM_ENGINE_GR: return NV2080_ENGINE_TYPE_GR0; + case NVKM_ENGINE_CE: return NV2080_ENGINE_TYPE_COPY0 + inst; + case NVKM_ENGINE_SEC2: return NV2080_ENGINE_TYPE_SEC2; + case NVKM_ENGINE_NVDEC: return NV2080_ENGINE_TYPE_NVDEC0 + inst; + case NVKM_ENGINE_NVENC: return NV2080_ENGINE_TYPE_NVENC0 + inst; + case NVKM_ENGINE_NVJPG: return NV2080_ENGINE_TYPE_NVJPEG0 + inst; + case NVKM_ENGINE_OFA: return NV2080_ENGINE_TYPE_OFA; + case NVKM_ENGINE_SW: return NV2080_ENGINE_TYPE_SW; + default: + break; + } + + WARN_ON(1); + return -EINVAL; +} + +static int +r535_fifo_engn_type(RM_ENGINE_TYPE rm, enum nvkm_subdev_type *ptype) +{ + switch (rm) { + case RM_ENGINE_TYPE_GR0: + *ptype = NVKM_ENGINE_GR; + return 0; + case RM_ENGINE_TYPE_COPY0...RM_ENGINE_TYPE_COPY9: + *ptype = NVKM_ENGINE_CE; + return rm - RM_ENGINE_TYPE_COPY0; + case RM_ENGINE_TYPE_NVDEC0...RM_ENGINE_TYPE_NVDEC7: + *ptype = NVKM_ENGINE_NVDEC; + return rm - RM_ENGINE_TYPE_NVDEC0; + case RM_ENGINE_TYPE_NVENC0...RM_ENGINE_TYPE_NVENC2: + *ptype = NVKM_ENGINE_NVENC; + return rm - RM_ENGINE_TYPE_NVENC0; + case RM_ENGINE_TYPE_SW: + *ptype = NVKM_ENGINE_SW; + return 0; + case RM_ENGINE_TYPE_SEC2: + *ptype = NVKM_ENGINE_SEC2; + return 0; + case RM_ENGINE_TYPE_NVJPEG0...RM_ENGINE_TYPE_NVJPEG7: + *ptype = NVKM_ENGINE_NVJPG; + return rm - RM_ENGINE_TYPE_NVJPEG0; + case RM_ENGINE_TYPE_OFA: + *ptype = NVKM_ENGINE_OFA; + return 0; + default: + return -EINVAL; + } +} + +static int +r535_fifo_ectx_size(struct nvkm_fifo *fifo) +{ + NV2080_CTRL_INTERNAL_GET_CONSTRUCTED_FALCON_INFO_PARAMS *ctrl; + struct nvkm_gsp *gsp = fifo->engine.subdev.device->gsp; + struct nvkm_runl *runl; + struct nvkm_engn *engn; + + ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice, + NV2080_CTRL_CMD_INTERNAL_GET_CONSTRUCTED_FALCON_INFO, + sizeof(*ctrl)); + if (WARN_ON(IS_ERR(ctrl))) + return PTR_ERR(ctrl); + + for (int i = 0; i < ctrl->numConstructedFalcons; i++) { + nvkm_runl_foreach(runl, fifo) { + nvkm_runl_foreach_engn(engn, runl) { + if (engn->rm.desc == ctrl->constructedFalconsTable[i].engDesc) { + engn->rm.size = + ctrl->constructedFalconsTable[i].ctxBufferSize; + break; + } + } + } + } + + nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); + return 0; +} + +static int +r535_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_subdev *subdev = &fifo->engine.subdev; + struct nvkm_gsp *gsp = subdev->device->gsp; + struct nvkm_runl *runl; + struct nvkm_engn *engn; + u32 cgids = 2048; + u32 chids = 2048; + int ret; + NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS *ctrl; + + if ((ret = nvkm_chid_new(&nvkm_chan_event, subdev, cgids, 0, cgids, &fifo->cgid)) || + (ret = nvkm_chid_new(&nvkm_chan_event, subdev, chids, 0, chids, &fifo->chid))) + return ret; + + ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice, + NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE, sizeof(*ctrl)); + if (WARN_ON(IS_ERR(ctrl))) + return PTR_ERR(ctrl); + + for (int i = 0; i < ctrl->numEntries; i++) { + const u32 addr = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST_PRI_BASE]; + const u32 id = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST]; + + runl = nvkm_runl_get(fifo, id, addr); + if (!runl) { + runl = nvkm_runl_new(fifo, id, addr, 0); + if (WARN_ON(IS_ERR(runl))) + continue; + } + } + + for (int i = 0; i < ctrl->numEntries; i++) { + const u32 addr = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST_PRI_BASE]; + const u32 rmid = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE]; + const u32 id = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST]; + enum nvkm_subdev_type type; + int inst, nv2080; + + runl = nvkm_runl_get(fifo, id, addr); + if (!runl) + continue; + + inst = r535_fifo_engn_type(rmid, &type); + if (inst < 0) { + nvkm_warn(subdev, "RM_ENGINE_TYPE 0x%x\n", rmid); + nvkm_runl_del(runl); + continue; + } + + nv2080 = r535_fifo_2080_type(type, inst); + if (nv2080 < 0) { + nvkm_runl_del(runl); + continue; + } + + switch (type) { + case NVKM_ENGINE_CE: + engn = nvkm_runl_add(runl, nv2080, &r535_ce, type, inst); + break; + case NVKM_ENGINE_GR: + engn = nvkm_runl_add(runl, nv2080, &r535_gr, type, inst); + break; + case NVKM_ENGINE_NVDEC: + case NVKM_ENGINE_NVENC: + case NVKM_ENGINE_NVJPG: + case NVKM_ENGINE_OFA: + engn = nvkm_runl_add(runl, nv2080, &r535_flcn, type, inst); + break; + case NVKM_ENGINE_SW: + continue; + default: + engn = NULL; + break; + } + + if (!engn) { + nvkm_runl_del(runl); + continue; + } + + engn->rm.desc = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_ENG_DESC]; + } + + nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); + + { + NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice, + NV2080_CTRL_CMD_CE_GET_FAULT_METHOD_BUFFER_SIZE, + sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + fifo->rm.mthdbuf_size = ctrl->size; + + nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); + } + + return r535_fifo_ectx_size(fifo); +} + +static void +r535_fifo_dtor(struct nvkm_fifo *fifo) +{ + kfree(fifo->func); +} + +int +r535_fifo_new(const struct nvkm_fifo_func *hw, struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) +{ + struct nvkm_fifo_func *rm; + + if (!(rm = kzalloc(sizeof(*rm), GFP_KERNEL))) + return -ENOMEM; + + rm->dtor = r535_fifo_dtor; + rm->runl_ctor = r535_fifo_runl_ctor; + rm->runl = &r535_runl; + rm->cgrp = hw->cgrp; + rm->cgrp.func = &r535_cgrp; + rm->chan = hw->chan; + rm->chan.func = &r535_chan; + rm->nonstall = &ga100_fifo_nonstall; + rm->nonstall_ctor = ga100_fifo_nonstall_ctor; + + return nvkm_fifo_new_(rm, device, type, inst, pfifo); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 5421321f8..19e6772ea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -18,6 +18,7 @@ struct nvkm_engn { bool (*mmu_fault_triggered)(struct nvkm_engn *); int (*ctor)(struct nvkm_engn *, struct nvkm_vctx *); void (*bind)(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); + int (*ctor2)(struct nvkm_engn *, struct nvkm_vctx *, struct nvkm_chan *); int (*ramht_add)(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *); void (*ramht_del)(struct nvkm_chan *, int hash); } *func; @@ -28,6 +29,11 @@ struct nvkm_engn { int fault; + struct { + u32 desc; + u32 size; + } rm; + struct list_head head; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index ea9e151db..1d39a6840 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -25,6 +25,7 @@ #include "runl.h" #include <core/memory.h> +#include <subdev/gsp.h> #include <subdev/mc.h> #include <subdev/vfn.h> @@ -282,5 +283,8 @@ int tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { + if (nvkm_gsp_rm(device->gsp)) + return r535_fifo_new(&tu102_fifo, device, type, inst, pfifo); + return nvkm_fifo_new_(&tu102_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 04140e011..9e56bcc16 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -317,6 +317,15 @@ nvkm_uchan = { .uevent = nvkm_uchan_uevent, }; +struct nvkm_chan * +nvkm_uchan_chan(struct nvkm_object *object) +{ + if (WARN_ON(object->func != &nvkm_uchan)) + return NULL; + + return nvkm_uchan(object)->chan; +} + int nvkm_uchan_new(struct nvkm_fifo *fifo, struct nvkm_cgrp *cgrp, const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) |