1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * uncore-frquency-tpmi: Uncore frequency scaling using TPMI
4 *
5 * Copyright (c) 2023, Intel Corporation.
6 * All Rights Reserved.
7 *
8 * The hardware interface to read/write is basically substitution of
9 * MSR 0x620 and 0x621.
10 * There are specific MMIO offset and bits to get/set minimum and
11 * maximum uncore ratio, similar to MSRs.
12 * The scope of the uncore MSRs was package scope. But TPMI allows
13 * new gen CPUs to have multiple uncore controls at uncore-cluster
14 * level. Each package can have multiple power domains which further
15 * can have multiple clusters.
16 * Here number of power domains = number of resources in this aux
17 * device. There are offsets and bits to discover number of clusters
18 * and offset for each cluster level controls.
19 *
20 */
21
22#include <linux/auxiliary_bus.h>
23#include <linux/bitfield.h>
24#include <linux/bits.h>
25#include <linux/io.h>
26#include <linux/module.h>
27#include <linux/intel_tpmi.h>
28
29#include "uncore-frequency-common.h"
30
31#define	UNCORE_MAJOR_VERSION		0
32#define	UNCORE_MINOR_VERSION		2
33#define UNCORE_HEADER_INDEX		0
34#define UNCORE_FABRIC_CLUSTER_OFFSET	8
35
36/* status + control + adv_ctl1 + adv_ctl2 */
37#define UNCORE_FABRIC_CLUSTER_SIZE	(4 * 8)
38
39#define UNCORE_STATUS_INDEX		0
40#define UNCORE_CONTROL_INDEX		8
41
42#define UNCORE_FREQ_KHZ_MULTIPLIER	100000
43
44struct tpmi_uncore_struct;
45
46/* Information for each cluster */
47struct tpmi_uncore_cluster_info {
48	bool root_domain;
49	u8 __iomem *cluster_base;
50	struct uncore_data uncore_data;
51	struct tpmi_uncore_struct *uncore_root;
52};
53
54/* Information for each power domain */
55struct tpmi_uncore_power_domain_info {
56	u8 __iomem *uncore_base;
57	int ufs_header_ver;
58	int cluster_count;
59	struct tpmi_uncore_cluster_info *cluster_infos;
60};
61
62/* Information for all power domains in a package */
63struct tpmi_uncore_struct {
64	int power_domain_count;
65	int max_ratio;
66	int min_ratio;
67	struct tpmi_uncore_power_domain_info *pd_info;
68	struct tpmi_uncore_cluster_info root_cluster;
69	bool write_blocked;
70};
71
72#define UNCORE_GENMASK_MIN_RATIO	GENMASK_ULL(21, 15)
73#define UNCORE_GENMASK_MAX_RATIO	GENMASK_ULL(14, 8)
74#define UNCORE_GENMASK_CURRENT_RATIO	GENMASK_ULL(6, 0)
75
76/* Helper function to read MMIO offset for max/min control frequency */
77static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info,
78			     unsigned int *min, unsigned int *max)
79{
80	u64 control;
81
82	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
83	*max = FIELD_GET(UNCORE_GENMASK_MAX_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
84	*min = FIELD_GET(UNCORE_GENMASK_MIN_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
85}
86
87#define UNCORE_MAX_RATIO	FIELD_MAX(UNCORE_GENMASK_MAX_RATIO)
88
89/* Callback for sysfs read for max/min frequencies. Called under mutex locks */
90static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
91				    unsigned int *max)
92{
93	struct tpmi_uncore_cluster_info *cluster_info;
94
95	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
96
97	if (cluster_info->root_domain) {
98		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
99		int i, _min = 0, _max = 0;
100
101		*min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER;
102		*max = 0;
103
104		/*
105		 * Get the max/min by looking at each cluster. Get the lowest
106		 * min and highest max.
107		 */
108		for (i = 0; i < uncore_root->power_domain_count; ++i) {
109			int j;
110
111			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) {
112				read_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
113						  &_min, &_max);
114				if (*min > _min)
115					*min = _min;
116				if (*max < _max)
117					*max = _max;
118			}
119		}
120		return 0;
121	}
122
123	read_control_freq(cluster_info, min, max);
124
125	return 0;
126}
127
128/* Helper function to write MMIO offset for max/min control frequency */
129static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input,
130			      unsigned int min_max)
131{
132	u64 control;
133
134	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
135
136	if (min_max) {
137		control &= ~UNCORE_GENMASK_MAX_RATIO;
138		control |= FIELD_PREP(UNCORE_GENMASK_MAX_RATIO, input);
139	} else {
140		control &= ~UNCORE_GENMASK_MIN_RATIO;
141		control |= FIELD_PREP(UNCORE_GENMASK_MIN_RATIO, input);
142	}
143
144	writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX));
145}
146
147/* Callback for sysfs write for max/min frequencies. Called under mutex locks */
148static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
149				     unsigned int min_max)
150{
151	struct tpmi_uncore_cluster_info *cluster_info;
152	struct tpmi_uncore_struct *uncore_root;
153
154	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
155	if (!input || input > UNCORE_MAX_RATIO)
156		return -EINVAL;
157
158	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
159	uncore_root = cluster_info->uncore_root;
160
161	if (uncore_root->write_blocked)
162		return -EPERM;
163
164	/* Update each cluster in a package */
165	if (cluster_info->root_domain) {
166		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
167		int i;
168
169		for (i = 0; i < uncore_root->power_domain_count; ++i) {
170			int j;
171
172			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j)
173				write_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
174						  input, min_max);
175		}
176
177		if (min_max)
178			uncore_root->max_ratio = input;
179		else
180			uncore_root->min_ratio = input;
181
182		return 0;
183	}
184
185	if (min_max && uncore_root->max_ratio && uncore_root->max_ratio < input)
186		return -EINVAL;
187
188	if (!min_max && uncore_root->min_ratio && uncore_root->min_ratio > input)
189		return -EINVAL;
190
191	write_control_freq(cluster_info, input, min_max);
192
193	return 0;
194}
195
196/* Callback for sysfs read for the current uncore frequency. Called under mutex locks */
197static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
198{
199	struct tpmi_uncore_cluster_info *cluster_info;
200	u64 status;
201
202	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
203	if (cluster_info->root_domain)
204		return -ENODATA;
205
206	status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
207	*freq = FIELD_GET(UNCORE_GENMASK_CURRENT_RATIO, status) * UNCORE_FREQ_KHZ_MULTIPLIER;
208
209	return 0;
210}
211
212static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore)
213{
214	int i;
215
216	for (i = 0; i < tpmi_uncore->power_domain_count; ++i) {
217		struct tpmi_uncore_power_domain_info *pd_info;
218		int j;
219
220		pd_info = &tpmi_uncore->pd_info[i];
221		if (!pd_info->uncore_base)
222			continue;
223
224		for (j = 0; j < pd_info->cluster_count; ++j) {
225			struct tpmi_uncore_cluster_info *cluster_info;
226
227			cluster_info = &pd_info->cluster_infos[j];
228			uncore_freq_remove_die_entry(&cluster_info->uncore_data);
229		}
230	}
231}
232
233#define UNCORE_VERSION_MASK			GENMASK_ULL(7, 0)
234#define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK	GENMASK_ULL(15, 8)
235#define UNCORE_CLUSTER_OFF_MASK			GENMASK_ULL(7, 0)
236#define UNCORE_MAX_CLUSTER_PER_DOMAIN		8
237
238static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
239{
240	bool read_blocked = 0, write_blocked = 0;
241	struct intel_tpmi_plat_info *plat_info;
242	struct tpmi_uncore_struct *tpmi_uncore;
243	bool uncore_sysfs_added = false;
244	int ret, i, pkg = 0;
245	int num_resources;
246
247	ret = tpmi_get_feature_status(auxdev, TPMI_ID_UNCORE, &read_blocked, &write_blocked);
248	if (ret)
249		dev_info(&auxdev->dev, "Can't read feature status: ignoring blocked status\n");
250
251	if (read_blocked) {
252		dev_info(&auxdev->dev, "Firmware has blocked reads, exiting\n");
253		return -ENODEV;
254	}
255
256	/* Get number of power domains, which is equal to number of resources */
257	num_resources = tpmi_get_resource_count(auxdev);
258	if (!num_resources)
259		return -EINVAL;
260
261	/* Register callbacks to uncore core */
262	ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
263				      uncore_read_freq);
264	if (ret)
265		return ret;
266
267	/* Allocate uncore instance per package */
268	tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL);
269	if (!tpmi_uncore) {
270		ret = -ENOMEM;
271		goto err_rem_common;
272	}
273
274	/* Allocate memory for all power domains in a package */
275	tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources,
276					    sizeof(*tpmi_uncore->pd_info),
277					    GFP_KERNEL);
278	if (!tpmi_uncore->pd_info) {
279		ret = -ENOMEM;
280		goto err_rem_common;
281	}
282
283	tpmi_uncore->power_domain_count = num_resources;
284	tpmi_uncore->write_blocked = write_blocked;
285
286	/* Get the package ID from the TPMI core */
287	plat_info = tpmi_get_platform_data(auxdev);
288	if (plat_info)
289		pkg = plat_info->package_id;
290	else
291		dev_info(&auxdev->dev, "Platform information is NULL\n");
292
293	for (i = 0; i < num_resources; ++i) {
294		struct tpmi_uncore_power_domain_info *pd_info;
295		struct resource *res;
296		u64 cluster_offset;
297		u8 cluster_mask;
298		int mask, j;
299		u64 header;
300
301		res = tpmi_get_resource_at_index(auxdev, i);
302		if (!res)
303			continue;
304
305		pd_info = &tpmi_uncore->pd_info[i];
306
307		pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res);
308		if (IS_ERR(pd_info->uncore_base)) {
309			ret = PTR_ERR(pd_info->uncore_base);
310			/*
311			 * Set to NULL so that clean up can still remove other
312			 * entries already created if any by
313			 * remove_cluster_entries()
314			 */
315			pd_info->uncore_base = NULL;
316			goto remove_clusters;
317		}
318
319		/* Check for version and skip this resource if there is mismatch */
320		header = readq(pd_info->uncore_base);
321		pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK;
322
323		if (pd_info->ufs_header_ver == TPMI_VERSION_INVALID)
324			continue;
325
326		if (TPMI_MAJOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MAJOR_VERSION) {
327			dev_err(&auxdev->dev, "Uncore: Unsupported major version:%lx\n",
328				TPMI_MAJOR_VERSION(pd_info->ufs_header_ver));
329			ret = -ENODEV;
330			goto remove_clusters;
331		}
332
333		if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) > UNCORE_MINOR_VERSION)
334			dev_info(&auxdev->dev, "Uncore: Ignore: Unsupported minor version:%lx\n",
335				 TPMI_MINOR_VERSION(pd_info->ufs_header_ver));
336
337		/* Get Cluster ID Mask */
338		cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header);
339		if (!cluster_mask) {
340			dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask);
341			continue;
342		}
343
344		/* Find out number of clusters in this resource */
345		pd_info->cluster_count = hweight8(cluster_mask);
346
347		pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count,
348						      sizeof(struct tpmi_uncore_cluster_info),
349						      GFP_KERNEL);
350		if (!pd_info->cluster_infos) {
351			ret = -ENOMEM;
352			goto remove_clusters;
353		}
354		/*
355		 * Each byte in the register point to status and control
356		 * registers belonging to cluster id 0-8.
357		 */
358		cluster_offset = readq(pd_info->uncore_base +
359					UNCORE_FABRIC_CLUSTER_OFFSET);
360
361		for (j = 0; j < pd_info->cluster_count; ++j) {
362			struct tpmi_uncore_cluster_info *cluster_info;
363
364			/* Get the offset for this cluster */
365			mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK);
366			/* Offset in QWORD, so change to bytes */
367			mask <<= 3;
368
369			cluster_info = &pd_info->cluster_infos[j];
370
371			cluster_info->cluster_base = pd_info->uncore_base + mask;
372
373			cluster_info->uncore_data.package_id = pkg;
374			/* There are no dies like Cascade Lake */
375			cluster_info->uncore_data.die_id = 0;
376			cluster_info->uncore_data.domain_id = i;
377			cluster_info->uncore_data.cluster_id = j;
378
379			cluster_info->uncore_root = tpmi_uncore;
380
381			ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0);
382			if (ret) {
383				cluster_info->cluster_base = NULL;
384				goto remove_clusters;
385			}
386			/* Point to next cluster offset */
387			cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN;
388			uncore_sysfs_added = true;
389		}
390	}
391
392	if (!uncore_sysfs_added) {
393		ret = -ENODEV;
394		goto remove_clusters;
395	}
396
397	auxiliary_set_drvdata(auxdev, tpmi_uncore);
398
399	tpmi_uncore->root_cluster.root_domain = true;
400	tpmi_uncore->root_cluster.uncore_root = tpmi_uncore;
401
402	tpmi_uncore->root_cluster.uncore_data.package_id = pkg;
403	tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID;
404	ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0);
405	if (ret)
406		goto remove_clusters;
407
408	return 0;
409
410remove_clusters:
411	remove_cluster_entries(tpmi_uncore);
412err_rem_common:
413	uncore_freq_common_exit();
414
415	return ret;
416}
417
418static void uncore_remove(struct auxiliary_device *auxdev)
419{
420	struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev);
421
422	uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data);
423	remove_cluster_entries(tpmi_uncore);
424
425	uncore_freq_common_exit();
426}
427
428static const struct auxiliary_device_id intel_uncore_id_table[] = {
429	{ .name = "intel_vsec.tpmi-uncore" },
430	{}
431};
432MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table);
433
434static struct auxiliary_driver intel_uncore_aux_driver = {
435	.id_table       = intel_uncore_id_table,
436	.remove         = uncore_remove,
437	.probe          = uncore_probe,
438};
439
440module_auxiliary_driver(intel_uncore_aux_driver);
441
442MODULE_IMPORT_NS(INTEL_TPMI);
443MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
444MODULE_DESCRIPTION("Intel TPMI UFS Driver");
445MODULE_LICENSE("GPL");
446