1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Early cpufeature override framework
4 *
5 * Copyright (C) 2020 Google LLC
6 * Author: Marc Zyngier <maz@kernel.org>
7 */
8
9#include <linux/ctype.h>
10#include <linux/kernel.h>
11#include <linux/libfdt.h>
12
13#include <asm/cacheflush.h>
14#include <asm/cpufeature.h>
15#include <asm/setup.h>
16
17#include "pi.h"
18
19#define FTR_DESC_NAME_LEN	20
20#define FTR_DESC_FIELD_LEN	10
21#define FTR_ALIAS_NAME_LEN	30
22#define FTR_ALIAS_OPTION_LEN	116
23
24static u64 __boot_status __initdata;
25
26typedef bool filter_t(u64 val);
27
28struct ftr_set_desc {
29	char 				name[FTR_DESC_NAME_LEN];
30	PREL64(struct arm64_ftr_override, override);
31	struct {
32		char			name[FTR_DESC_FIELD_LEN];
33		u8			shift;
34		u8			width;
35		PREL64(filter_t,	filter);
36	} 				fields[];
37};
38
39#define FIELD(n, s, f)	{ .name = n, .shift = s, .width = 4, .filter = f }
40
41static bool __init mmfr1_vh_filter(u64 val)
42{
43	/*
44	 * If we ever reach this point while running VHE, we're
45	 * guaranteed to be on one of these funky, VHE-stuck CPUs. If
46	 * the user was trying to force nVHE on us, proceed with
47	 * attitude adjustment.
48	 */
49	return !(__boot_status == (BOOT_CPU_FLAG_E2H | BOOT_CPU_MODE_EL2) &&
50		 val == 0);
51}
52
53static const struct ftr_set_desc mmfr1 __prel64_initconst = {
54	.name		= "id_aa64mmfr1",
55	.override	= &id_aa64mmfr1_override,
56	.fields		= {
57		FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter),
58		{}
59	},
60};
61
62
63static bool __init mmfr2_varange_filter(u64 val)
64{
65	int __maybe_unused feat;
66
67	if (val)
68		return false;
69
70#ifdef CONFIG_ARM64_LPA2
71	feat = cpuid_feature_extract_signed_field(read_sysreg(id_aa64mmfr0_el1),
72						  ID_AA64MMFR0_EL1_TGRAN_SHIFT);
73	if (feat >= ID_AA64MMFR0_EL1_TGRAN_LPA2) {
74		id_aa64mmfr0_override.val |=
75			(ID_AA64MMFR0_EL1_TGRAN_LPA2 - 1) << ID_AA64MMFR0_EL1_TGRAN_SHIFT;
76		id_aa64mmfr0_override.mask |= 0xfU << ID_AA64MMFR0_EL1_TGRAN_SHIFT;
77	}
78#endif
79	return true;
80}
81
82static const struct ftr_set_desc mmfr2 __prel64_initconst = {
83	.name		= "id_aa64mmfr2",
84	.override	= &id_aa64mmfr2_override,
85	.fields		= {
86		FIELD("varange", ID_AA64MMFR2_EL1_VARange_SHIFT, mmfr2_varange_filter),
87		{}
88	},
89};
90
91static bool __init pfr0_sve_filter(u64 val)
92{
93	/*
94	 * Disabling SVE also means disabling all the features that
95	 * are associated with it. The easiest way to do it is just to
96	 * override id_aa64zfr0_el1 to be 0.
97	 */
98	if (!val) {
99		id_aa64zfr0_override.val = 0;
100		id_aa64zfr0_override.mask = GENMASK(63, 0);
101	}
102
103	return true;
104}
105
106static const struct ftr_set_desc pfr0 __prel64_initconst = {
107	.name		= "id_aa64pfr0",
108	.override	= &id_aa64pfr0_override,
109	.fields		= {
110	        FIELD("sve", ID_AA64PFR0_EL1_SVE_SHIFT, pfr0_sve_filter),
111		FIELD("el0", ID_AA64PFR0_EL1_EL0_SHIFT, NULL),
112		{}
113	},
114};
115
116static bool __init pfr1_sme_filter(u64 val)
117{
118	/*
119	 * Similarly to SVE, disabling SME also means disabling all
120	 * the features that are associated with it. Just set
121	 * id_aa64smfr0_el1 to 0 and don't look back.
122	 */
123	if (!val) {
124		id_aa64smfr0_override.val = 0;
125		id_aa64smfr0_override.mask = GENMASK(63, 0);
126	}
127
128	return true;
129}
130
131static const struct ftr_set_desc pfr1 __prel64_initconst = {
132	.name		= "id_aa64pfr1",
133	.override	= &id_aa64pfr1_override,
134	.fields		= {
135		FIELD("bt", ID_AA64PFR1_EL1_BT_SHIFT, NULL ),
136		FIELD("mte", ID_AA64PFR1_EL1_MTE_SHIFT, NULL),
137		FIELD("sme", ID_AA64PFR1_EL1_SME_SHIFT, pfr1_sme_filter),
138		{}
139	},
140};
141
142static const struct ftr_set_desc isar1 __prel64_initconst = {
143	.name		= "id_aa64isar1",
144	.override	= &id_aa64isar1_override,
145	.fields		= {
146		FIELD("gpi", ID_AA64ISAR1_EL1_GPI_SHIFT, NULL),
147		FIELD("gpa", ID_AA64ISAR1_EL1_GPA_SHIFT, NULL),
148		FIELD("api", ID_AA64ISAR1_EL1_API_SHIFT, NULL),
149		FIELD("apa", ID_AA64ISAR1_EL1_APA_SHIFT, NULL),
150		{}
151	},
152};
153
154static const struct ftr_set_desc isar2 __prel64_initconst = {
155	.name		= "id_aa64isar2",
156	.override	= &id_aa64isar2_override,
157	.fields		= {
158		FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL),
159		FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL),
160		FIELD("mops", ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL),
161		{}
162	},
163};
164
165static const struct ftr_set_desc smfr0 __prel64_initconst = {
166	.name		= "id_aa64smfr0",
167	.override	= &id_aa64smfr0_override,
168	.fields		= {
169		FIELD("smever", ID_AA64SMFR0_EL1_SMEver_SHIFT, NULL),
170		/* FA64 is a one bit field... :-/ */
171		{ "fa64", ID_AA64SMFR0_EL1_FA64_SHIFT, 1, },
172		{}
173	},
174};
175
176static bool __init hvhe_filter(u64 val)
177{
178	u64 mmfr1 = read_sysreg(id_aa64mmfr1_el1);
179
180	return (val == 1 &&
181		lower_32_bits(__boot_status) == BOOT_CPU_MODE_EL2 &&
182		cpuid_feature_extract_unsigned_field(mmfr1,
183						     ID_AA64MMFR1_EL1_VH_SHIFT));
184}
185
186static const struct ftr_set_desc sw_features __prel64_initconst = {
187	.name		= "arm64_sw",
188	.override	= &arm64_sw_feature_override,
189	.fields		= {
190		FIELD("nokaslr", ARM64_SW_FEATURE_OVERRIDE_NOKASLR, NULL),
191		FIELD("hvhe", ARM64_SW_FEATURE_OVERRIDE_HVHE, hvhe_filter),
192		FIELD("rodataoff", ARM64_SW_FEATURE_OVERRIDE_RODATA_OFF, NULL),
193		{}
194	},
195};
196
197static const
198PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = {
199	{ &mmfr1	},
200	{ &mmfr2	},
201	{ &pfr0 	},
202	{ &pfr1 	},
203	{ &isar1	},
204	{ &isar2	},
205	{ &smfr0	},
206	{ &sw_features	},
207};
208
209static const struct {
210	char	alias[FTR_ALIAS_NAME_LEN];
211	char	feature[FTR_ALIAS_OPTION_LEN];
212} aliases[] __initconst = {
213	{ "kvm_arm.mode=nvhe",		"arm64_sw.hvhe=0 id_aa64mmfr1.vh=0" },
214	{ "kvm_arm.mode=protected",	"arm64_sw.hvhe=1" },
215	{ "arm64.nosve",		"id_aa64pfr0.sve=0" },
216	{ "arm64.nosme",		"id_aa64pfr1.sme=0" },
217	{ "arm64.nobti",		"id_aa64pfr1.bt=0" },
218	{ "arm64.nopauth",
219	  "id_aa64isar1.gpi=0 id_aa64isar1.gpa=0 "
220	  "id_aa64isar1.api=0 id_aa64isar1.apa=0 "
221	  "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0"	   },
222	{ "arm64.nomops",		"id_aa64isar2.mops=0" },
223	{ "arm64.nomte",		"id_aa64pfr1.mte=0" },
224	{ "nokaslr",			"arm64_sw.nokaslr=1" },
225	{ "rodata=off",			"arm64_sw.rodataoff=1" },
226	{ "arm64.nolva",		"id_aa64mmfr2.varange=0" },
227	{ "arm64.no32bit_el0",		"id_aa64pfr0.el0=1" },
228};
229
230static int __init parse_hexdigit(const char *p, u64 *v)
231{
232	// skip "0x" if it comes next
233	if (p[0] == '0' && tolower(p[1]) == 'x')
234		p += 2;
235
236	// check whether the RHS is a single hex digit
237	if (!isxdigit(p[0]) || (p[1] && !isspace(p[1])))
238		return -EINVAL;
239
240	*v = tolower(*p) - (isdigit(*p) ? '0' : 'a' - 10);
241	return 0;
242}
243
244static int __init find_field(const char *cmdline, char *opt, int len,
245			     const struct ftr_set_desc *reg, int f, u64 *v)
246{
247	int flen = strlen(reg->fields[f].name);
248
249	// append '<fieldname>=' to obtain '<name>.<fieldname>='
250	memcpy(opt + len, reg->fields[f].name, flen);
251	len += flen;
252	opt[len++] = '=';
253
254	if (memcmp(cmdline, opt, len))
255		return -1;
256
257	return parse_hexdigit(cmdline + len, v);
258}
259
260static void __init match_options(const char *cmdline)
261{
262	char opt[FTR_DESC_NAME_LEN + FTR_DESC_FIELD_LEN + 2];
263	int i;
264
265	for (i = 0; i < ARRAY_SIZE(regs); i++) {
266		const struct ftr_set_desc *reg = prel64_pointer(regs[i].reg);
267		struct arm64_ftr_override *override;
268		int len = strlen(reg->name);
269		int f;
270
271		override = prel64_pointer(reg->override);
272
273		// set opt[] to '<name>.'
274		memcpy(opt, reg->name, len);
275		opt[len++] = '.';
276
277		for (f = 0; reg->fields[f].name[0] != '\0'; f++) {
278			u64 shift = reg->fields[f].shift;
279			u64 width = reg->fields[f].width ?: 4;
280			u64 mask = GENMASK_ULL(shift + width - 1, shift);
281			bool (*filter)(u64 val);
282			u64 v;
283
284			if (find_field(cmdline, opt, len, reg, f, &v))
285				continue;
286
287			/*
288			 * If an override gets filtered out, advertise
289			 * it by setting the value to the all-ones while
290			 * clearing the mask... Yes, this is fragile.
291			 */
292			filter = prel64_pointer(reg->fields[f].filter);
293			if (filter && !filter(v)) {
294				override->val  |= mask;
295				override->mask &= ~mask;
296				continue;
297			}
298
299			override->val  &= ~mask;
300			override->val  |= (v << shift) & mask;
301			override->mask |= mask;
302
303			return;
304		}
305	}
306}
307
308static __init void __parse_cmdline(const char *cmdline, bool parse_aliases)
309{
310	do {
311		char buf[256];
312		size_t len;
313		int i;
314
315		cmdline = skip_spaces(cmdline);
316
317		/* terminate on "--" appearing on the command line by itself */
318		if (cmdline[0] == '-' && cmdline[1] == '-' && isspace(cmdline[2]))
319			return;
320
321		for (len = 0; cmdline[len] && !isspace(cmdline[len]); len++) {
322			if (len >= sizeof(buf) - 1)
323				break;
324			if (cmdline[len] == '-')
325				buf[len] = '_';
326			else
327				buf[len] = cmdline[len];
328		}
329		if (!len)
330			return;
331
332		buf[len] = 0;
333
334		cmdline += len;
335
336		match_options(buf);
337
338		for (i = 0; parse_aliases && i < ARRAY_SIZE(aliases); i++)
339			if (!memcmp(buf, aliases[i].alias, len + 1))
340				__parse_cmdline(aliases[i].feature, false);
341	} while (1);
342}
343
344static __init const u8 *get_bootargs_cmdline(const void *fdt, int node)
345{
346	static char const bootargs[] __initconst = "bootargs";
347	const u8 *prop;
348
349	if (node < 0)
350		return NULL;
351
352	prop = fdt_getprop(fdt, node, bootargs, NULL);
353	if (!prop)
354		return NULL;
355
356	return strlen(prop) ? prop : NULL;
357}
358
359static __init void parse_cmdline(const void *fdt, int chosen)
360{
361	static char const cmdline[] __initconst = CONFIG_CMDLINE;
362	const u8 *prop = get_bootargs_cmdline(fdt, chosen);
363
364	if (IS_ENABLED(CONFIG_CMDLINE_FORCE) || !prop)
365		__parse_cmdline(cmdline, true);
366
367	if (!IS_ENABLED(CONFIG_CMDLINE_FORCE) && prop)
368		__parse_cmdline(prop, true);
369}
370
371void __init init_feature_override(u64 boot_status, const void *fdt,
372				  int chosen)
373{
374	struct arm64_ftr_override *override;
375	const struct ftr_set_desc *reg;
376	int i;
377
378	for (i = 0; i < ARRAY_SIZE(regs); i++) {
379		reg = prel64_pointer(regs[i].reg);
380		override = prel64_pointer(reg->override);
381
382		override->val  = 0;
383		override->mask = 0;
384	}
385
386	__boot_status = boot_status;
387
388	parse_cmdline(fdt, chosen);
389
390	for (i = 0; i < ARRAY_SIZE(regs); i++) {
391		reg = prel64_pointer(regs[i].reg);
392		override = prel64_pointer(reg->override);
393		dcache_clean_inval_poc((unsigned long)override,
394				       (unsigned long)(override + 1));
395	}
396}
397
398char * __init skip_spaces(const char *str)
399{
400	while (isspace(*str))
401		++str;
402	return (char *)str;
403}
404