summaryrefslogtreecommitdiffstats
path: root/drivers/accel/habanalabs/common/context.c
blob: 9c8b1b37b510ce822c8408bf821c8adab7f5c917 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2016-2021 HabanaLabs, Ltd.
 * All Rights Reserved.
 */

#include "habanalabs.h"

#include <linux/slab.h>

static void encaps_handle_do_release(struct hl_cs_encaps_sig_handle *handle, bool put_hw_sob,
					bool put_ctx)
{
	struct hl_encaps_signals_mgr *mgr = &handle->ctx->sig_mgr;

	if (put_hw_sob)
		hw_sob_put(handle->hw_sob);

	spin_lock(&mgr->lock);
	idr_remove(&mgr->handles, handle->id);
	spin_unlock(&mgr->lock);

	if (put_ctx)
		hl_ctx_put(handle->ctx);

	kfree(handle);
}

void hl_encaps_release_handle_and_put_ctx(struct kref *ref)
{
	struct hl_cs_encaps_sig_handle *handle =
			container_of(ref, struct hl_cs_encaps_sig_handle, refcount);

	encaps_handle_do_release(handle, false, true);
}

static void hl_encaps_release_handle_and_put_sob(struct kref *ref)
{
	struct hl_cs_encaps_sig_handle *handle =
			container_of(ref, struct hl_cs_encaps_sig_handle, refcount);

	encaps_handle_do_release(handle, true, false);
}

void hl_encaps_release_handle_and_put_sob_ctx(struct kref *ref)
{
	struct hl_cs_encaps_sig_handle *handle =
			container_of(ref, struct hl_cs_encaps_sig_handle, refcount);

	encaps_handle_do_release(handle, true, true);
}

static void hl_encaps_sig_mgr_init(struct hl_encaps_signals_mgr *mgr)
{
	spin_lock_init(&mgr->lock);
	idr_init(&mgr->handles);
}

static void hl_encaps_sig_mgr_fini(struct hl_device *hdev, struct hl_encaps_signals_mgr *mgr)
{
	struct hl_cs_encaps_sig_handle *handle;
	struct idr *idp;
	u32 id;

	idp = &mgr->handles;

	/* The IDR is expected to be empty at this stage, because any left signal should have been
	 * released as part of CS roll-back.
	 */
	if (!idr_is_empty(idp)) {
		dev_warn(hdev->dev,
			"device released while some encaps signals handles are still allocated\n");
		idr_for_each_entry(idp, handle, id)
			kref_put(&handle->refcount, hl_encaps_release_handle_and_put_sob);
	}

	idr_destroy(&mgr->handles);
}

static void hl_ctx_fini(struct hl_ctx *ctx)
{
	struct hl_device *hdev = ctx->hdev;
	int i;

	/* Release all allocated HW block mapped list entries and destroy
	 * the mutex.
	 */
	hl_hw_block_mem_fini(ctx);

	/*
	 * If we arrived here, there are no jobs waiting for this context
	 * on its queues so we can safely remove it.
	 * This is because for each CS, we increment the ref count and for
	 * every CS that was finished we decrement it and we won't arrive
	 * to this function unless the ref count is 0
	 */

	for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
		hl_fence_put(ctx->cs_pending[i]);

	kfree(ctx->cs_pending);

	if (ctx->asid != HL_KERNEL_ASID_ID) {
		dev_dbg(hdev->dev, "closing user context %d\n", ctx->asid);

		/* The engines are stopped as there is no executing CS, but the
		 * Coresight might be still working by accessing addresses
		 * related to the stopped engines. Hence stop it explicitly.
		 */
		if (hdev->in_debug)
			hl_device_set_debug_mode(hdev, ctx, false);

		hdev->asic_funcs->ctx_fini(ctx);

		hl_dec_ctx_fini(ctx);

		hl_cb_va_pool_fini(ctx);
		hl_vm_ctx_fini(ctx);
		hl_asid_free(hdev, ctx->asid);
		hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);
	} else {
		dev_dbg(hdev->dev, "closing kernel context\n");
		hdev->asic_funcs->ctx_fini(ctx);
		hl_vm_ctx_fini(ctx);
		hl_mmu_ctx_fini(ctx);
	}
}

void hl_ctx_do_release(struct kref *ref)
{
	struct hl_ctx *ctx;

	ctx = container_of(ref, struct hl_ctx, refcount);

	hl_ctx_fini(ctx);

	if (ctx->hpriv) {
		struct hl_fpriv *hpriv = ctx->hpriv;

		mutex_lock(&hpriv->ctx_lock);
		hpriv->ctx = NULL;
		mutex_unlock(&hpriv->ctx_lock);

		hl_hpriv_put(hpriv);
	}

	kfree(ctx);
}

int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv)
{
	struct hl_ctx_mgr *ctx_mgr = &hpriv->ctx_mgr;
	struct hl_ctx *ctx;
	int rc;

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx) {
		rc = -ENOMEM;
		goto out_err;
	}

	mutex_lock(&ctx_mgr->lock);
	rc = idr_alloc(&ctx_mgr->handles, ctx, 1, 0, GFP_KERNEL);
	mutex_unlock(&ctx_mgr->lock);

	if (rc < 0) {
		dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");
		goto free_ctx;
	}

	ctx->handle = rc;

	rc = hl_ctx_init(hdev, ctx, false);
	if (rc)
		goto remove_from_idr;

	hl_hpriv_get(hpriv);
	ctx->hpriv = hpriv;

	/* TODO: remove for multiple contexts per process */
	hpriv->ctx = ctx;

	/* TODO: remove the following line for multiple process support */
	hdev->is_compute_ctx_active = true;

	return 0;

remove_from_idr:
	mutex_lock(&ctx_mgr->lock);
	idr_remove(&ctx_mgr->handles, ctx->handle);
	mutex_unlock(&ctx_mgr->lock);
free_ctx:
	kfree(ctx);
out_err:
	return rc;
}

int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
{
	int rc = 0, i;

	ctx->hdev = hdev;

	kref_init(&ctx->refcount);

	ctx->cs_sequence = 1;
	spin_lock_init(&ctx->cs_lock);
	atomic_set(&ctx->thread_ctx_switch_token, 1);
	ctx->thread_ctx_switch_wait_token = 0;
	ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
				sizeof(struct hl_fence *),
				GFP_KERNEL);
	if (!ctx->cs_pending)
		return -ENOMEM;

	INIT_LIST_HEAD(&ctx->outcome_store.used_list);
	INIT_LIST_HEAD(&ctx->outcome_store.free_list);
	hash_init(ctx->outcome_store.outcome_map);
	for (i = 0; i < ARRAY_SIZE(ctx->outcome_store.nodes_pool); ++i)
		list_add(&ctx->outcome_store.nodes_pool[i].list_link,
			 &ctx->outcome_store.free_list);

	hl_hw_block_mem_init(ctx);

	if (is_kernel_ctx) {
		ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */
		rc = hl_vm_ctx_init(ctx);
		if (rc) {
			dev_err(hdev->dev, "Failed to init mem ctx module\n");
			rc = -ENOMEM;
			goto err_hw_block_mem_fini;
		}

		rc = hdev->asic_funcs->ctx_init(ctx);
		if (rc) {
			dev_err(hdev->dev, "ctx_init failed\n");
			goto err_vm_ctx_fini;
		}
	} else {
		ctx->asid = hl_asid_alloc(hdev);
		if (!ctx->asid) {
			dev_err(hdev->dev, "No free ASID, failed to create context\n");
			rc = -ENOMEM;
			goto err_hw_block_mem_fini;
		}

		rc = hl_vm_ctx_init(ctx);
		if (rc) {
			dev_err(hdev->dev, "Failed to init mem ctx module\n");
			rc = -ENOMEM;
			goto err_asid_free;
		}

		rc = hl_cb_va_pool_init(ctx);
		if (rc) {
			dev_err(hdev->dev,
				"Failed to init VA pool for mapped CB\n");
			goto err_vm_ctx_fini;
		}

		rc = hdev->asic_funcs->ctx_init(ctx);
		if (rc) {
			dev_err(hdev->dev, "ctx_init failed\n");
			goto err_cb_va_pool_fini;
		}

		hl_encaps_sig_mgr_init(&ctx->sig_mgr);

		dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
	}

	return 0;

err_cb_va_pool_fini:
	hl_cb_va_pool_fini(ctx);
err_vm_ctx_fini:
	hl_vm_ctx_fini(ctx);
err_asid_free:
	if (ctx->asid != HL_KERNEL_ASID_ID)
		hl_asid_free(hdev, ctx->asid);
err_hw_block_mem_fini:
	hl_hw_block_mem_fini(ctx);
	kfree(ctx->cs_pending);

	return rc;
}

static int hl_ctx_get_unless_zero(struct hl_ctx *ctx)
{
	return kref_get_unless_zero(&ctx->refcount);
}

void hl_ctx_get(struct hl_ctx *ctx)
{
	kref_get(&ctx->refcount);
}

int hl_ctx_put(struct hl_ctx *ctx)
{
	return kref_put(&ctx->refcount, hl_ctx_do_release);
}

struct hl_ctx *hl_get_compute_ctx(struct hl_device *hdev)
{
	struct hl_ctx *ctx = NULL;
	struct hl_fpriv *hpriv;

	mutex_lock(&hdev->fpriv_list_lock);

	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
		mutex_lock(&hpriv->ctx_lock);
		ctx = hpriv->ctx;
		if (ctx && !hl_ctx_get_unless_zero(ctx))
			ctx = NULL;
		mutex_unlock(&hpriv->ctx_lock);

		/* There can only be a single user which has opened the compute device, so exit
		 * immediately once we find its context or if we see that it has been released
		 */
		break;
	}

	mutex_unlock(&hdev->fpriv_list_lock);

	return ctx;
}

/*
 * hl_ctx_get_fence_locked - get CS fence under CS lock
 *
 * @ctx: pointer to the context structure.
 * @seq: CS sequences number
 *
 * @return valid fence pointer on success, NULL if fence is gone, otherwise
 *         error pointer.
 *
 * NOTE: this function shall be called with cs_lock locked
 */
static struct hl_fence *hl_ctx_get_fence_locked(struct hl_ctx *ctx, u64 seq)
{
	struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
	struct hl_fence *fence;

	if (seq >= ctx->cs_sequence)
		return ERR_PTR(-EINVAL);

	if (seq + asic_prop->max_pending_cs < ctx->cs_sequence)
		return NULL;

	fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
	hl_fence_get(fence);
	return fence;
}

struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
{
	struct hl_fence *fence;

	spin_lock(&ctx->cs_lock);

	fence = hl_ctx_get_fence_locked(ctx, seq);

	spin_unlock(&ctx->cs_lock);

	return fence;
}

/*
 * hl_ctx_get_fences - get multiple CS fences under the same CS lock
 *
 * @ctx: pointer to the context structure.
 * @seq_arr: array of CS sequences to wait for
 * @fence: fence array to store the CS fences
 * @arr_len: length of seq_arr and fence_arr
 *
 * @return 0 on success, otherwise non 0 error code
 */
int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
				struct hl_fence **fence, u32 arr_len)
{
	struct hl_fence **fence_arr_base = fence;
	int i, rc = 0;

	spin_lock(&ctx->cs_lock);

	for (i = 0; i < arr_len; i++, fence++) {
		u64 seq = seq_arr[i];

		*fence = hl_ctx_get_fence_locked(ctx, seq);

		if (IS_ERR(*fence)) {
			dev_err(ctx->hdev->dev,
				"Failed to get fence for CS with seq 0x%llx\n",
					seq);
			rc = PTR_ERR(*fence);
			break;
		}
	}

	spin_unlock(&ctx->cs_lock);

	if (rc)
		hl_fences_put(fence_arr_base, i);

	return rc;
}

/*
 * hl_ctx_mgr_init - initialize the context manager
 *
 * @ctx_mgr: pointer to context manager structure
 *
 * This manager is an object inside the hpriv object of the user process.
 * The function is called when a user process opens the FD.
 */
void hl_ctx_mgr_init(struct hl_ctx_mgr *ctx_mgr)
{
	mutex_init(&ctx_mgr->lock);
	idr_init(&ctx_mgr->handles);
}

/*
 * hl_ctx_mgr_fini - finalize the context manager
 *
 * @hdev: pointer to device structure
 * @ctx_mgr: pointer to context manager structure
 *
 * This function goes over all the contexts in the manager and frees them.
 * It is called when a process closes the FD.
 */
void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *ctx_mgr)
{
	struct hl_ctx *ctx;
	struct idr *idp;
	u32 id;

	idp = &ctx_mgr->handles;

	idr_for_each_entry(idp, ctx, id)
		kref_put(&ctx->refcount, hl_ctx_do_release);

	idr_destroy(&ctx_mgr->handles);
	mutex_destroy(&ctx_mgr->lock);
}