summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/nouveau/nouveau_sched.c
blob: 3b7ea52212260583ffbdfae77ddd92936d1d00ae (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
// SPDX-License-Identifier: MIT

#include <linux/slab.h>
#include <drm/gpu_scheduler.h>
#include <drm/drm_syncobj.h>

#include "nouveau_drv.h"
#include "nouveau_gem.h"
#include "nouveau_mem.h"
#include "nouveau_dma.h"
#include "nouveau_exec.h"
#include "nouveau_abi16.h"
#include "nouveau_sched.h"

/* FIXME
 *
 * We want to make sure that jobs currently executing can't be deferred by
 * other jobs competing for the hardware. Otherwise we might end up with job
 * timeouts just because of too many clients submitting too many jobs. We don't
 * want jobs to time out because of system load, but because of the job being
 * too bulky.
 *
 * For now allow for up to 16 concurrent jobs in flight until we know how many
 * rings the hardware can process in parallel.
 */
#define NOUVEAU_SCHED_HW_SUBMISSIONS		16
#define NOUVEAU_SCHED_JOB_TIMEOUT_MS		10000

int
nouveau_job_init(struct nouveau_job *job,
		 struct nouveau_job_args *args)
{
	struct nouveau_sched_entity *entity = args->sched_entity;
	int ret;

	job->file_priv = args->file_priv;
	job->cli = nouveau_cli(args->file_priv);
	job->entity = entity;

	job->sync = args->sync;
	job->resv_usage = args->resv_usage;

	job->ops = args->ops;

	job->in_sync.count = args->in_sync.count;
	if (job->in_sync.count) {
		if (job->sync)
			return -EINVAL;

		job->in_sync.data = kmemdup(args->in_sync.s,
					 sizeof(*args->in_sync.s) *
					 args->in_sync.count,
					 GFP_KERNEL);
		if (!job->in_sync.data)
			return -ENOMEM;
	}

	job->out_sync.count = args->out_sync.count;
	if (job->out_sync.count) {
		if (job->sync) {
			ret = -EINVAL;
			goto err_free_in_sync;
		}

		job->out_sync.data = kmemdup(args->out_sync.s,
					  sizeof(*args->out_sync.s) *
					  args->out_sync.count,
					  GFP_KERNEL);
		if (!job->out_sync.data) {
			ret = -ENOMEM;
			goto err_free_in_sync;
		}

		job->out_sync.objs = kcalloc(job->out_sync.count,
					     sizeof(*job->out_sync.objs),
					     GFP_KERNEL);
		if (!job->out_sync.objs) {
			ret = -ENOMEM;
			goto err_free_out_sync;
		}

		job->out_sync.chains = kcalloc(job->out_sync.count,
					       sizeof(*job->out_sync.chains),
					       GFP_KERNEL);
		if (!job->out_sync.chains) {
			ret = -ENOMEM;
			goto err_free_objs;
		}

	}

	ret = drm_sched_job_init(&job->base, &entity->base, NULL);
	if (ret)
		goto err_free_chains;

	job->state = NOUVEAU_JOB_INITIALIZED;

	return 0;

err_free_chains:
	kfree(job->out_sync.chains);
err_free_objs:
	kfree(job->out_sync.objs);
err_free_out_sync:
	kfree(job->out_sync.data);
err_free_in_sync:
	kfree(job->in_sync.data);
return ret;
}

void
nouveau_job_free(struct nouveau_job *job)
{
	kfree(job->in_sync.data);
	kfree(job->out_sync.data);
	kfree(job->out_sync.objs);
	kfree(job->out_sync.chains);
}

void nouveau_job_fini(struct nouveau_job *job)
{
	dma_fence_put(job->done_fence);
	drm_sched_job_cleanup(&job->base);
	job->ops->free(job);
}

static int
sync_find_fence(struct nouveau_job *job,
		struct drm_nouveau_sync *sync,
		struct dma_fence **fence)
{
	u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK;
	u64 point = 0;
	int ret;

	if (stype != DRM_NOUVEAU_SYNC_SYNCOBJ &&
	    stype != DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ)
		return -EOPNOTSUPP;

	if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ)
		point = sync->timeline_value;

	ret = drm_syncobj_find_fence(job->file_priv,
				     sync->handle, point,
				     0 /* flags */, fence);
	if (ret)
		return ret;

	return 0;
}

static int
nouveau_job_add_deps(struct nouveau_job *job)
{
	struct dma_fence *in_fence = NULL;
	int ret, i;

	for (i = 0; i < job->in_sync.count; i++) {
		struct drm_nouveau_sync *sync = &job->in_sync.data[i];

		ret = sync_find_fence(job, sync, &in_fence);
		if (ret) {
			NV_PRINTK(warn, job->cli,
				  "Failed to find syncobj (-> in): handle=%d\n",
				  sync->handle);
			return ret;
		}

		ret = drm_sched_job_add_dependency(&job->base, in_fence);
		if (ret)
			return ret;
	}

	return 0;
}

static void
nouveau_job_fence_attach_cleanup(struct nouveau_job *job)
{
	int i;

	for (i = 0; i < job->out_sync.count; i++) {
		struct drm_syncobj *obj = job->out_sync.objs[i];
		struct dma_fence_chain *chain = job->out_sync.chains[i];

		if (obj)
			drm_syncobj_put(obj);

		if (chain)
			dma_fence_chain_free(chain);
	}
}

static int
nouveau_job_fence_attach_prepare(struct nouveau_job *job)
{
	int i, ret;

	for (i = 0; i < job->out_sync.count; i++) {
		struct drm_nouveau_sync *sync = &job->out_sync.data[i];
		struct drm_syncobj **pobj = &job->out_sync.objs[i];
		struct dma_fence_chain **pchain = &job->out_sync.chains[i];
		u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK;

		if (stype != DRM_NOUVEAU_SYNC_SYNCOBJ &&
		    stype != DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) {
			ret = -EINVAL;
			goto err_sync_cleanup;
		}

		*pobj = drm_syncobj_find(job->file_priv, sync->handle);
		if (!*pobj) {
			NV_PRINTK(warn, job->cli,
				  "Failed to find syncobj (-> out): handle=%d\n",
				  sync->handle);
			ret = -ENOENT;
			goto err_sync_cleanup;
		}

		if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) {
			*pchain = dma_fence_chain_alloc();
			if (!*pchain) {
				ret = -ENOMEM;
				goto err_sync_cleanup;
			}
		}
	}

	return 0;

err_sync_cleanup:
	nouveau_job_fence_attach_cleanup(job);
	return ret;
}

static void
nouveau_job_fence_attach(struct nouveau_job *job)
{
	struct dma_fence *fence = job->done_fence;
	int i;

	for (i = 0; i < job->out_sync.count; i++) {
		struct drm_nouveau_sync *sync = &job->out_sync.data[i];
		struct drm_syncobj **pobj = &job->out_sync.objs[i];
		struct dma_fence_chain **pchain = &job->out_sync.chains[i];
		u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK;

		if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) {
			drm_syncobj_add_point(*pobj, *pchain, fence,
					      sync->timeline_value);
		} else {
			drm_syncobj_replace_fence(*pobj, fence);
		}

		drm_syncobj_put(*pobj);
		*pobj = NULL;
		*pchain = NULL;
	}
}

int
nouveau_job_submit(struct nouveau_job *job)
{
	struct nouveau_sched_entity *entity = to_nouveau_sched_entity(job->base.entity);
	struct dma_fence *done_fence = NULL;
	int ret;

	ret = nouveau_job_add_deps(job);
	if (ret)
		goto err;

	ret = nouveau_job_fence_attach_prepare(job);
	if (ret)
		goto err;

	/* Make sure the job appears on the sched_entity's queue in the same
	 * order as it was submitted.
	 */
	mutex_lock(&entity->mutex);

	/* Guarantee we won't fail after the submit() callback returned
	 * successfully.
	 */
	if (job->ops->submit) {
		ret = job->ops->submit(job);
		if (ret)
			goto err_cleanup;
	}

	drm_sched_job_arm(&job->base);
	job->done_fence = dma_fence_get(&job->base.s_fence->finished);
	if (job->sync)
		done_fence = dma_fence_get(job->done_fence);

	/* If a sched job depends on a dma-fence from a job from the same GPU
	 * scheduler instance, but a different scheduler entity, the GPU
	 * scheduler does only wait for the particular job to be scheduled,
	 * rather than for the job to fully complete. This is due to the GPU
	 * scheduler assuming that there is a scheduler instance per ring.
	 * However, the current implementation, in order to avoid arbitrary
	 * amounts of kthreads, has a single scheduler instance while scheduler
	 * entities represent rings.
	 *
	 * As a workaround, set the DRM_SCHED_FENCE_DONT_PIPELINE for all
	 * out-fences in order to force the scheduler to wait for full job
	 * completion for dependent jobs from different entities and same
	 * scheduler instance.
	 *
	 * There is some work in progress [1] to address the issues of firmware
	 * schedulers; once it is in-tree the scheduler topology in Nouveau
	 * should be re-worked accordingly.
	 *
	 * [1] https://lore.kernel.org/dri-devel/20230801205103.627779-1-matthew.brost@intel.com/
	 */
	set_bit(DRM_SCHED_FENCE_DONT_PIPELINE, &job->done_fence->flags);

	if (job->ops->armed_submit)
		job->ops->armed_submit(job);

	nouveau_job_fence_attach(job);

	/* Set job state before pushing the job to the scheduler,
	 * such that we do not overwrite the job state set in run().
	 */
	job->state = NOUVEAU_JOB_SUBMIT_SUCCESS;

	drm_sched_entity_push_job(&job->base);

	mutex_unlock(&entity->mutex);

	if (done_fence) {
		dma_fence_wait(done_fence, true);
		dma_fence_put(done_fence);
	}

	return 0;

err_cleanup:
	mutex_unlock(&entity->mutex);
	nouveau_job_fence_attach_cleanup(job);
err:
	job->state = NOUVEAU_JOB_SUBMIT_FAILED;
	return ret;
}

bool
nouveau_sched_entity_qwork(struct nouveau_sched_entity *entity,
			   struct work_struct *work)
{
	return queue_work(entity->sched_wq, work);
}

static struct dma_fence *
nouveau_job_run(struct nouveau_job *job)
{
	struct dma_fence *fence;

	fence = job->ops->run(job);
	if (IS_ERR(fence))
		job->state = NOUVEAU_JOB_RUN_FAILED;
	else
		job->state = NOUVEAU_JOB_RUN_SUCCESS;

	return fence;
}

static struct dma_fence *
nouveau_sched_run_job(struct drm_sched_job *sched_job)
{
	struct nouveau_job *job = to_nouveau_job(sched_job);

	return nouveau_job_run(job);
}

static enum drm_gpu_sched_stat
nouveau_sched_timedout_job(struct drm_sched_job *sched_job)
{
	struct drm_gpu_scheduler *sched = sched_job->sched;
	struct nouveau_job *job = to_nouveau_job(sched_job);
	enum drm_gpu_sched_stat stat = DRM_GPU_SCHED_STAT_NOMINAL;

	drm_sched_stop(sched, sched_job);

	if (job->ops->timeout)
		stat = job->ops->timeout(job);
	else
		NV_PRINTK(warn, job->cli, "Generic job timeout.\n");

	drm_sched_start(sched, true);

	return stat;
}

static void
nouveau_sched_free_job(struct drm_sched_job *sched_job)
{
	struct nouveau_job *job = to_nouveau_job(sched_job);

	nouveau_job_fini(job);
}

int nouveau_sched_entity_init(struct nouveau_sched_entity *entity,
			      struct drm_gpu_scheduler *sched,
			      struct workqueue_struct *sched_wq)
{
	mutex_init(&entity->mutex);
	spin_lock_init(&entity->job.list.lock);
	INIT_LIST_HEAD(&entity->job.list.head);
	init_waitqueue_head(&entity->job.wq);

	entity->sched_wq = sched_wq;
	return drm_sched_entity_init(&entity->base,
				     DRM_SCHED_PRIORITY_NORMAL,
				     &sched, 1, NULL);
}

void
nouveau_sched_entity_fini(struct nouveau_sched_entity *entity)
{
	drm_sched_entity_destroy(&entity->base);
}

static const struct drm_sched_backend_ops nouveau_sched_ops = {
	.run_job = nouveau_sched_run_job,
	.timedout_job = nouveau_sched_timedout_job,
	.free_job = nouveau_sched_free_job,
};

int nouveau_sched_init(struct nouveau_drm *drm)
{
	struct drm_gpu_scheduler *sched = &drm->sched;
	long job_hang_limit = msecs_to_jiffies(NOUVEAU_SCHED_JOB_TIMEOUT_MS);

	drm->sched_wq = create_singlethread_workqueue("nouveau_sched_wq");
	if (!drm->sched_wq)
		return -ENOMEM;

	return drm_sched_init(sched, &nouveau_sched_ops,
			      NOUVEAU_SCHED_HW_SUBMISSIONS, 0, job_hang_limit,
			      NULL, NULL, "nouveau_sched", drm->dev->dev);
}

void nouveau_sched_fini(struct nouveau_drm *drm)
{
	destroy_workqueue(drm->sched_wq);
	drm_sched_fini(&drm->sched);
}