summaryrefslogtreecommitdiffstats
path: root/arch/mips/kvm/tlb.c
blob: 4e91971daae143158f1814dad40acb5ed09687bf (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
 * TLB handlers run from KSEG0
 *
 * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
 * Authors: Sanjay Lal <sanjayl@kymasys.com>
 */

#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/kvm_host.h>
#include <linux/srcu.h>

#include <asm/cpu.h>
#include <asm/bootinfo.h>
#include <asm/mipsregs.h>
#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
#include <asm/tlb.h>
#include <asm/tlbdebug.h>

#undef CONFIG_MIPS_MT
#include <asm/r4kcache.h>
#define CONFIG_MIPS_MT

unsigned long GUESTID_MASK;
EXPORT_SYMBOL_GPL(GUESTID_MASK);
unsigned long GUESTID_FIRST_VERSION;
EXPORT_SYMBOL_GPL(GUESTID_FIRST_VERSION);
unsigned long GUESTID_VERSION_MASK;
EXPORT_SYMBOL_GPL(GUESTID_VERSION_MASK);

static u32 kvm_mips_get_root_asid(struct kvm_vcpu *vcpu)
{
	struct mm_struct *gpa_mm = &vcpu->kvm->arch.gpa_mm;

	if (cpu_has_guestid)
		return 0;
	else
		return cpu_asid(smp_processor_id(), gpa_mm);
}

static int _kvm_mips_host_tlb_inv(unsigned long entryhi)
{
	int idx;

	write_c0_entryhi(entryhi);
	mtc0_tlbw_hazard();

	tlb_probe();
	tlb_probe_hazard();
	idx = read_c0_index();

	BUG_ON(idx >= current_cpu_data.tlbsize);

	if (idx >= 0) {
		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
		write_c0_entrylo0(0);
		write_c0_entrylo1(0);
		mtc0_tlbw_hazard();

		tlb_write_indexed();
		tlbw_use_hazard();
	}

	return idx;
}

/* GuestID management */

/**
 * clear_root_gid() - Set GuestCtl1.RID for normal root operation.
 */
static inline void clear_root_gid(void)
{
	if (cpu_has_guestid) {
		clear_c0_guestctl1(MIPS_GCTL1_RID);
		mtc0_tlbw_hazard();
	}
}

/**
 * set_root_gid_to_guest_gid() - Set GuestCtl1.RID to match GuestCtl1.ID.
 *
 * Sets the root GuestID to match the current guest GuestID, for TLB operation
 * on the GPA->RPA mappings in the root TLB.
 *
 * The caller must be sure to disable HTW while the root GID is set, and
 * possibly longer if TLB registers are modified.
 */
static inline void set_root_gid_to_guest_gid(void)
{
	unsigned int guestctl1;

	if (cpu_has_guestid) {
		back_to_back_c0_hazard();
		guestctl1 = read_c0_guestctl1();
		guestctl1 = (guestctl1 & ~MIPS_GCTL1_RID) |
			((guestctl1 & MIPS_GCTL1_ID) >> MIPS_GCTL1_ID_SHIFT)
						     << MIPS_GCTL1_RID_SHIFT;
		write_c0_guestctl1(guestctl1);
		mtc0_tlbw_hazard();
	}
}

int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
{
	int idx;
	unsigned long flags, old_entryhi;

	local_irq_save(flags);
	htw_stop();

	/* Set root GuestID for root probe and write of guest TLB entry */
	set_root_gid_to_guest_gid();

	old_entryhi = read_c0_entryhi();

	idx = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
				     kvm_mips_get_root_asid(vcpu));

	write_c0_entryhi(old_entryhi);
	clear_root_gid();
	mtc0_tlbw_hazard();

	htw_start();
	local_irq_restore(flags);

	/*
	 * We don't want to get reserved instruction exceptions for missing tlb
	 * entries.
	 */
	if (cpu_has_vtag_icache)
		flush_icache_all();

	if (idx > 0)
		kvm_debug("%s: Invalidated root entryhi %#lx @ idx %d\n",
			  __func__, (va & VPN2_MASK) |
				    kvm_mips_get_root_asid(vcpu), idx);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vz_host_tlb_inv);

/**
 * kvm_vz_guest_tlb_lookup() - Lookup a guest VZ TLB mapping.
 * @vcpu:	KVM VCPU pointer.
 * @gpa:	Guest virtual address in a TLB mapped guest segment.
 * @gpa:	Pointer to output guest physical address it maps to.
 *
 * Converts a guest virtual address in a guest TLB mapped segment to a guest
 * physical address, by probing the guest TLB.
 *
 * Returns:	0 if guest TLB mapping exists for @gva. *@gpa will have been
 *		written.
 *		-EFAULT if no guest TLB mapping exists for @gva. *@gpa may not
 *		have been written.
 */
int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva,
			    unsigned long *gpa)
{
	unsigned long o_entryhi, o_entrylo[2], o_pagemask;
	unsigned int o_index;
	unsigned long entrylo[2], pagemask, pagemaskbit, pa;
	unsigned long flags;
	int index;

	/* Probe the guest TLB for a mapping */
	local_irq_save(flags);
	/* Set root GuestID for root probe of guest TLB entry */
	htw_stop();
	set_root_gid_to_guest_gid();

	o_entryhi = read_gc0_entryhi();
	o_index = read_gc0_index();

	write_gc0_entryhi((o_entryhi & 0x3ff) | (gva & ~0xfffl));
	mtc0_tlbw_hazard();
	guest_tlb_probe();
	tlb_probe_hazard();

	index = read_gc0_index();
	if (index < 0) {
		/* No match, fail */
		write_gc0_entryhi(o_entryhi);
		write_gc0_index(o_index);

		clear_root_gid();
		htw_start();
		local_irq_restore(flags);
		return -EFAULT;
	}

	/* Match! read the TLB entry */
	o_entrylo[0] = read_gc0_entrylo0();
	o_entrylo[1] = read_gc0_entrylo1();
	o_pagemask = read_gc0_pagemask();

	mtc0_tlbr_hazard();
	guest_tlb_read();
	tlb_read_hazard();

	entrylo[0] = read_gc0_entrylo0();
	entrylo[1] = read_gc0_entrylo1();
	pagemask = ~read_gc0_pagemask() & ~0x1fffl;

	write_gc0_entryhi(o_entryhi);
	write_gc0_index(o_index);
	write_gc0_entrylo0(o_entrylo[0]);
	write_gc0_entrylo1(o_entrylo[1]);
	write_gc0_pagemask(o_pagemask);

	clear_root_gid();
	htw_start();
	local_irq_restore(flags);

	/* Select one of the EntryLo values and interpret the GPA */
	pagemaskbit = (pagemask ^ (pagemask & (pagemask - 1))) >> 1;
	pa = entrylo[!!(gva & pagemaskbit)];

	/*
	 * TLB entry may have become invalid since TLB probe if physical FTLB
	 * entries are shared between threads (e.g. I6400).
	 */
	if (!(pa & ENTRYLO_V))
		return -EFAULT;

	/*
	 * Note, this doesn't take guest MIPS32 XPA into account, where PFN is
	 * split with XI/RI in the middle.
	 */
	pa = (pa << 6) & ~0xfffl;
	pa |= gva & ~(pagemask | pagemaskbit);

	*gpa = pa;
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_vz_guest_tlb_lookup);

/**
 * kvm_vz_local_flush_roottlb_all_guests() - Flush all root TLB entries for
 * guests.
 *
 * Invalidate all entries in root tlb which are GPA mappings.
 */
void kvm_vz_local_flush_roottlb_all_guests(void)
{
	unsigned long flags;
	unsigned long old_entryhi, old_pagemask, old_guestctl1;
	int entry;

	if (WARN_ON(!cpu_has_guestid))
		return;

	local_irq_save(flags);
	htw_stop();

	/* TLBR may clobber EntryHi.ASID, PageMask, and GuestCtl1.RID */
	old_entryhi = read_c0_entryhi();
	old_pagemask = read_c0_pagemask();
	old_guestctl1 = read_c0_guestctl1();

	/*
	 * Invalidate guest entries in root TLB while leaving root entries
	 * intact when possible.
	 */
	for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
		write_c0_index(entry);
		mtc0_tlbw_hazard();
		tlb_read();
		tlb_read_hazard();

		/* Don't invalidate non-guest (RVA) mappings in the root TLB */
		if (!(read_c0_guestctl1() & MIPS_GCTL1_RID))
			continue;

		/* Make sure all entries differ. */
		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
		write_c0_entrylo0(0);
		write_c0_entrylo1(0);
		write_c0_guestctl1(0);
		mtc0_tlbw_hazard();
		tlb_write_indexed();
	}

	write_c0_entryhi(old_entryhi);
	write_c0_pagemask(old_pagemask);
	write_c0_guestctl1(old_guestctl1);
	tlbw_use_hazard();

	htw_start();
	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_vz_local_flush_roottlb_all_guests);

/**
 * kvm_vz_local_flush_guesttlb_all() - Flush all guest TLB entries.
 *
 * Invalidate all entries in guest tlb irrespective of guestid.
 */
void kvm_vz_local_flush_guesttlb_all(void)
{
	unsigned long flags;
	unsigned long old_index;
	unsigned long old_entryhi;
	unsigned long old_entrylo[2];
	unsigned long old_pagemask;
	int entry;
	u64 cvmmemctl2 = 0;

	local_irq_save(flags);

	/* Preserve all clobbered guest registers */
	old_index = read_gc0_index();
	old_entryhi = read_gc0_entryhi();
	old_entrylo[0] = read_gc0_entrylo0();
	old_entrylo[1] = read_gc0_entrylo1();
	old_pagemask = read_gc0_pagemask();

	switch (current_cpu_type()) {
	case CPU_CAVIUM_OCTEON3:
		/* Inhibit machine check due to multiple matching TLB entries */
		cvmmemctl2 = read_c0_cvmmemctl2();
		cvmmemctl2 |= CVMMEMCTL2_INHIBITTS;
		write_c0_cvmmemctl2(cvmmemctl2);
		break;
	}

	/* Invalidate guest entries in guest TLB */
	write_gc0_entrylo0(0);
	write_gc0_entrylo1(0);
	write_gc0_pagemask(0);
	for (entry = 0; entry < current_cpu_data.guest.tlbsize; entry++) {
		/* Make sure all entries differ. */
		write_gc0_index(entry);
		write_gc0_entryhi(UNIQUE_GUEST_ENTRYHI(entry));
		mtc0_tlbw_hazard();
		guest_tlb_write_indexed();
	}

	if (cvmmemctl2) {
		cvmmemctl2 &= ~CVMMEMCTL2_INHIBITTS;
		write_c0_cvmmemctl2(cvmmemctl2);
	}

	write_gc0_index(old_index);
	write_gc0_entryhi(old_entryhi);
	write_gc0_entrylo0(old_entrylo[0]);
	write_gc0_entrylo1(old_entrylo[1]);
	write_gc0_pagemask(old_pagemask);
	tlbw_use_hazard();

	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_vz_local_flush_guesttlb_all);

/**
 * kvm_vz_save_guesttlb() - Save a range of guest TLB entries.
 * @buf:	Buffer to write TLB entries into.
 * @index:	Start index.
 * @count:	Number of entries to save.
 *
 * Save a range of guest TLB entries. The caller must ensure interrupts are
 * disabled.
 */
void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index,
			  unsigned int count)
{
	unsigned int end = index + count;
	unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
	unsigned int guestctl1 = 0;
	int old_index, i;

	/* Save registers we're about to clobber */
	old_index = read_gc0_index();
	old_entryhi = read_gc0_entryhi();
	old_entrylo0 = read_gc0_entrylo0();
	old_entrylo1 = read_gc0_entrylo1();
	old_pagemask = read_gc0_pagemask();

	/* Set root GuestID for root probe */
	htw_stop();
	set_root_gid_to_guest_gid();
	if (cpu_has_guestid)
		guestctl1 = read_c0_guestctl1();

	/* Read each entry from guest TLB */
	for (i = index; i < end; ++i, ++buf) {
		write_gc0_index(i);

		mtc0_tlbr_hazard();
		guest_tlb_read();
		tlb_read_hazard();

		if (cpu_has_guestid &&
		    (read_c0_guestctl1() ^ guestctl1) & MIPS_GCTL1_RID) {
			/* Entry invalid or belongs to another guest */
			buf->tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
			buf->tlb_lo[0] = 0;
			buf->tlb_lo[1] = 0;
			buf->tlb_mask = 0;
		} else {
			/* Entry belongs to the right guest */
			buf->tlb_hi = read_gc0_entryhi();
			buf->tlb_lo[0] = read_gc0_entrylo0();
			buf->tlb_lo[1] = read_gc0_entrylo1();
			buf->tlb_mask = read_gc0_pagemask();
		}
	}

	/* Clear root GuestID again */
	clear_root_gid();
	htw_start();

	/* Restore clobbered registers */
	write_gc0_index(old_index);
	write_gc0_entryhi(old_entryhi);
	write_gc0_entrylo0(old_entrylo0);
	write_gc0_entrylo1(old_entrylo1);
	write_gc0_pagemask(old_pagemask);

	tlbw_use_hazard();
}
EXPORT_SYMBOL_GPL(kvm_vz_save_guesttlb);

/**
 * kvm_vz_load_guesttlb() - Save a range of guest TLB entries.
 * @buf:	Buffer to read TLB entries from.
 * @index:	Start index.
 * @count:	Number of entries to load.
 *
 * Load a range of guest TLB entries. The caller must ensure interrupts are
 * disabled.
 */
void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index,
			  unsigned int count)
{
	unsigned int end = index + count;
	unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
	int old_index, i;

	/* Save registers we're about to clobber */
	old_index = read_gc0_index();
	old_entryhi = read_gc0_entryhi();
	old_entrylo0 = read_gc0_entrylo0();
	old_entrylo1 = read_gc0_entrylo1();
	old_pagemask = read_gc0_pagemask();

	/* Set root GuestID for root probe */
	htw_stop();
	set_root_gid_to_guest_gid();

	/* Write each entry to guest TLB */
	for (i = index; i < end; ++i, ++buf) {
		write_gc0_index(i);
		write_gc0_entryhi(buf->tlb_hi);
		write_gc0_entrylo0(buf->tlb_lo[0]);
		write_gc0_entrylo1(buf->tlb_lo[1]);
		write_gc0_pagemask(buf->tlb_mask);

		mtc0_tlbw_hazard();
		guest_tlb_write_indexed();
	}

	/* Clear root GuestID again */
	clear_root_gid();
	htw_start();

	/* Restore clobbered registers */
	write_gc0_index(old_index);
	write_gc0_entryhi(old_entryhi);
	write_gc0_entrylo0(old_entrylo0);
	write_gc0_entrylo1(old_entrylo1);
	write_gc0_pagemask(old_pagemask);

	tlbw_use_hazard();
}
EXPORT_SYMBOL_GPL(kvm_vz_load_guesttlb);

#ifdef CONFIG_CPU_LOONGSON64
void kvm_loongson_clear_guest_vtlb(void)
{
	int idx = read_gc0_index();

	/* Set root GuestID for root probe and write of guest TLB entry */
	set_root_gid_to_guest_gid();

	write_gc0_index(0);
	guest_tlbinvf();
	write_gc0_index(idx);

	clear_root_gid();
	set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
}
EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_vtlb);

void kvm_loongson_clear_guest_ftlb(void)
{
	int i;
	int idx = read_gc0_index();

	/* Set root GuestID for root probe and write of guest TLB entry */
	set_root_gid_to_guest_gid();

	for (i = current_cpu_data.tlbsizevtlb;
	     i < (current_cpu_data.tlbsizevtlb +
		     current_cpu_data.tlbsizeftlbsets);
	     i++) {
		write_gc0_index(i);
		guest_tlbinvf();
	}
	write_gc0_index(idx);

	clear_root_gid();
	set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
}
EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_ftlb);
#endif