summaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/cpu/tsx.c
blob: b31ee4f1657a82d15e7df449fbcca1952f8bec56 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
// SPDX-License-Identifier: GPL-2.0
/*
 * Intel Transactional Synchronization Extensions (TSX) control.
 *
 * Copyright (C) 2019-2021 Intel Corporation
 *
 * Author:
 *	Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
 */

#include <linux/cpufeature.h>

#include <asm/cmdline.h>
#include <asm/cpu.h>

#include "cpu.h"

#undef pr_fmt
#define pr_fmt(fmt) "tsx: " fmt

enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED;

static void tsx_disable(void)
{
	u64 tsx;

	rdmsrl(MSR_IA32_TSX_CTRL, tsx);

	/* Force all transactions to immediately abort */
	tsx |= TSX_CTRL_RTM_DISABLE;

	/*
	 * Ensure TSX support is not enumerated in CPUID.
	 * This is visible to userspace and will ensure they
	 * do not waste resources trying TSX transactions that
	 * will always abort.
	 */
	tsx |= TSX_CTRL_CPUID_CLEAR;

	wrmsrl(MSR_IA32_TSX_CTRL, tsx);
}

static void tsx_enable(void)
{
	u64 tsx;

	rdmsrl(MSR_IA32_TSX_CTRL, tsx);

	/* Enable the RTM feature in the cpu */
	tsx &= ~TSX_CTRL_RTM_DISABLE;

	/*
	 * Ensure TSX support is enumerated in CPUID.
	 * This is visible to userspace and will ensure they
	 * can enumerate and use the TSX feature.
	 */
	tsx &= ~TSX_CTRL_CPUID_CLEAR;

	wrmsrl(MSR_IA32_TSX_CTRL, tsx);
}

static enum tsx_ctrl_states x86_get_tsx_auto_mode(void)
{
	if (boot_cpu_has_bug(X86_BUG_TAA))
		return TSX_CTRL_DISABLE;

	return TSX_CTRL_ENABLE;
}

/*
 * Disabling TSX is not a trivial business.
 *
 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT
 * which says that TSX is practically disabled (all transactions are
 * aborted by default). When that bit is set, the kernel unconditionally
 * disables TSX.
 *
 * In order to do that, however, it needs to dance a bit:
 *
 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and
 * the MSR is present only when *two* CPUID bits are set:
 *
 * - X86_FEATURE_RTM_ALWAYS_ABORT
 * - X86_FEATURE_TSX_FORCE_ABORT
 *
 * 2. The second method is for CPUs which do not have the above-mentioned
 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX
 * through that one. Those CPUs can also have the initially mentioned
 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy
 * applies: TSX gets disabled unconditionally.
 *
 * When either of the two methods are present, the kernel disables TSX and
 * clears the respective RTM and HLE feature flags.
 *
 * An additional twist in the whole thing presents late microcode loading
 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID
 * bit to be set after the update.
 *
 * A subsequent hotplug operation on any logical CPU except the BSP will
 * cause for the supported CPUID feature bits to get re-detected and, if
 * RTM and HLE get cleared all of a sudden, but, userspace did consult
 * them before the update, then funny explosions will happen. Long story
 * short: the kernel doesn't modify CPUID feature bits after booting.
 *
 * That's why, this function's call in init_intel() doesn't clear the
 * feature flags.
 */
static void tsx_clear_cpuid(void)
{
	u64 msr;

	/*
	 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID
	 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present.
	 */
	if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) &&
	    boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
		rdmsrl(MSR_TSX_FORCE_ABORT, msr);
		msr |= MSR_TFA_TSX_CPUID_CLEAR;
		wrmsrl(MSR_TSX_FORCE_ABORT, msr);
	} else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) {
		rdmsrl(MSR_IA32_TSX_CTRL, msr);
		msr |= TSX_CTRL_CPUID_CLEAR;
		wrmsrl(MSR_IA32_TSX_CTRL, msr);
	}
}

/*
 * Disable TSX development mode
 *
 * When the microcode released in Feb 2022 is applied, TSX will be disabled by
 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123
 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is
 * not recommended for production deployments. In particular, applying MD_CLEAR
 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient
 * execution attack may not be effective on these processors when Intel TSX is
 * enabled with updated microcode.
 */
static void tsx_dev_mode_disable(void)
{
	u64 mcu_opt_ctrl;

	/* Check if RTM_ALLOW exists */
	if (!boot_cpu_has_bug(X86_BUG_TAA) ||
	    !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) ||
	    !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL))
		return;

	rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);

	if (mcu_opt_ctrl & RTM_ALLOW) {
		mcu_opt_ctrl &= ~RTM_ALLOW;
		wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
		setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT);
	}
}

void __init tsx_init(void)
{
	char arg[5] = {};
	int ret;

	tsx_dev_mode_disable();

	/*
	 * Hardware will always abort a TSX transaction when the CPUID bit
	 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate
	 * CPUID.RTM and CPUID.HLE bits. Clear them here.
	 */
	if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) {
		tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT;
		tsx_clear_cpuid();
		setup_clear_cpu_cap(X86_FEATURE_RTM);
		setup_clear_cpu_cap(X86_FEATURE_HLE);
		return;
	}

	/*
	 * TSX is controlled via MSR_IA32_TSX_CTRL.  However, support for this
	 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES.
	 *
	 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a
	 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES
	 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get
	 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus,
	 * tsx= cmdline requests will do nothing on CPUs without
	 * MSR_IA32_TSX_CTRL support.
	 */
	if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) {
		setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL);
	} else {
		tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED;
		return;
	}

	ret = cmdline_find_option(boot_command_line, "tsx", arg, sizeof(arg));
	if (ret >= 0) {
		if (!strcmp(arg, "on")) {
			tsx_ctrl_state = TSX_CTRL_ENABLE;
		} else if (!strcmp(arg, "off")) {
			tsx_ctrl_state = TSX_CTRL_DISABLE;
		} else if (!strcmp(arg, "auto")) {
			tsx_ctrl_state = x86_get_tsx_auto_mode();
		} else {
			tsx_ctrl_state = TSX_CTRL_DISABLE;
			pr_err("invalid option, defaulting to off\n");
		}
	} else {
		/* tsx= not provided */
		if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO))
			tsx_ctrl_state = x86_get_tsx_auto_mode();
		else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF))
			tsx_ctrl_state = TSX_CTRL_DISABLE;
		else
			tsx_ctrl_state = TSX_CTRL_ENABLE;
	}

	if (tsx_ctrl_state == TSX_CTRL_DISABLE) {
		tsx_disable();

		/*
		 * tsx_disable() will change the state of the RTM and HLE CPUID
		 * bits. Clear them here since they are now expected to be not
		 * set.
		 */
		setup_clear_cpu_cap(X86_FEATURE_RTM);
		setup_clear_cpu_cap(X86_FEATURE_HLE);
	} else if (tsx_ctrl_state == TSX_CTRL_ENABLE) {

		/*
		 * HW defaults TSX to be enabled at bootup.
		 * We may still need the TSX enable support
		 * during init for special cases like
		 * kexec after TSX is disabled.
		 */
		tsx_enable();

		/*
		 * tsx_enable() will change the state of the RTM and HLE CPUID
		 * bits. Force them here since they are now expected to be set.
		 */
		setup_force_cpu_cap(X86_FEATURE_RTM);
		setup_force_cpu_cap(X86_FEATURE_HLE);
	}
}

void tsx_ap_init(void)
{
	tsx_dev_mode_disable();

	if (tsx_ctrl_state == TSX_CTRL_ENABLE)
		tsx_enable();
	else if (tsx_ctrl_state == TSX_CTRL_DISABLE)
		tsx_disable();
	else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT)
		/* See comment over that function for more details. */
		tsx_clear_cpuid();
}