1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * uncore-frquency-tpmi: Uncore frequency scaling using TPMI 4 * 5 * Copyright (c) 2023, Intel Corporation. 6 * All Rights Reserved. 7 * 8 * The hardware interface to read/write is basically substitution of 9 * MSR 0x620 and 0x621. 10 * There are specific MMIO offset and bits to get/set minimum and 11 * maximum uncore ratio, similar to MSRs. 12 * The scope of the uncore MSRs was package scope. But TPMI allows 13 * new gen CPUs to have multiple uncore controls at uncore-cluster 14 * level. Each package can have multiple power domains which further 15 * can have multiple clusters. 16 * Here number of power domains = number of resources in this aux 17 * device. There are offsets and bits to discover number of clusters 18 * and offset for each cluster level controls. 19 * 20 */ 21 22#include <linux/auxiliary_bus.h> 23#include <linux/bitfield.h> 24#include <linux/bits.h> 25#include <linux/io.h> 26#include <linux/module.h> 27#include <linux/intel_tpmi.h> 28 29#include "uncore-frequency-common.h" 30 31#define UNCORE_MAJOR_VERSION 0 32#define UNCORE_MINOR_VERSION 2 33#define UNCORE_HEADER_INDEX 0 34#define UNCORE_FABRIC_CLUSTER_OFFSET 8 35 36/* status + control + adv_ctl1 + adv_ctl2 */ 37#define UNCORE_FABRIC_CLUSTER_SIZE (4 * 8) 38 39#define UNCORE_STATUS_INDEX 0 40#define UNCORE_CONTROL_INDEX 8 41 42#define UNCORE_FREQ_KHZ_MULTIPLIER 100000 43 44struct tpmi_uncore_struct; 45 46/* Information for each cluster */ 47struct tpmi_uncore_cluster_info { 48 bool root_domain; 49 u8 __iomem *cluster_base; 50 struct uncore_data uncore_data; 51 struct tpmi_uncore_struct *uncore_root; 52}; 53 54/* Information for each power domain */ 55struct tpmi_uncore_power_domain_info { 56 u8 __iomem *uncore_base; 57 int ufs_header_ver; 58 int cluster_count; 59 struct tpmi_uncore_cluster_info *cluster_infos; 60}; 61 62/* Information for all power domains in a package */ 63struct tpmi_uncore_struct { 64 int power_domain_count; 65 int max_ratio; 66 int min_ratio; 67 struct tpmi_uncore_power_domain_info *pd_info; 68 struct tpmi_uncore_cluster_info root_cluster; 69 bool write_blocked; 70}; 71 72#define UNCORE_GENMASK_MIN_RATIO GENMASK_ULL(21, 15) 73#define UNCORE_GENMASK_MAX_RATIO GENMASK_ULL(14, 8) 74#define UNCORE_GENMASK_CURRENT_RATIO GENMASK_ULL(6, 0) 75 76/* Helper function to read MMIO offset for max/min control frequency */ 77static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info, 78 unsigned int *min, unsigned int *max) 79{ 80 u64 control; 81 82 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 83 *max = FIELD_GET(UNCORE_GENMASK_MAX_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER; 84 *min = FIELD_GET(UNCORE_GENMASK_MIN_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER; 85} 86 87#define UNCORE_MAX_RATIO FIELD_MAX(UNCORE_GENMASK_MAX_RATIO) 88 89/* Callback for sysfs read for max/min frequencies. Called under mutex locks */ 90static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min, 91 unsigned int *max) 92{ 93 struct tpmi_uncore_cluster_info *cluster_info; 94 95 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 96 97 if (cluster_info->root_domain) { 98 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root; 99 int i, _min = 0, _max = 0; 100 101 *min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER; 102 *max = 0; 103 104 /* 105 * Get the max/min by looking at each cluster. Get the lowest 106 * min and highest max. 107 */ 108 for (i = 0; i < uncore_root->power_domain_count; ++i) { 109 int j; 110 111 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) { 112 read_control_freq(&uncore_root->pd_info[i].cluster_infos[j], 113 &_min, &_max); 114 if (*min > _min) 115 *min = _min; 116 if (*max < _max) 117 *max = _max; 118 } 119 } 120 return 0; 121 } 122 123 read_control_freq(cluster_info, min, max); 124 125 return 0; 126} 127 128/* Helper function to write MMIO offset for max/min control frequency */ 129static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input, 130 unsigned int min_max) 131{ 132 u64 control; 133 134 control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX); 135 136 if (min_max) { 137 control &= ~UNCORE_GENMASK_MAX_RATIO; 138 control |= FIELD_PREP(UNCORE_GENMASK_MAX_RATIO, input); 139 } else { 140 control &= ~UNCORE_GENMASK_MIN_RATIO; 141 control |= FIELD_PREP(UNCORE_GENMASK_MIN_RATIO, input); 142 } 143 144 writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX)); 145} 146 147/* Callback for sysfs write for max/min frequencies. Called under mutex locks */ 148static int uncore_write_control_freq(struct uncore_data *data, unsigned int input, 149 unsigned int min_max) 150{ 151 struct tpmi_uncore_cluster_info *cluster_info; 152 struct tpmi_uncore_struct *uncore_root; 153 154 input /= UNCORE_FREQ_KHZ_MULTIPLIER; 155 if (!input || input > UNCORE_MAX_RATIO) 156 return -EINVAL; 157 158 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 159 uncore_root = cluster_info->uncore_root; 160 161 if (uncore_root->write_blocked) 162 return -EPERM; 163 164 /* Update each cluster in a package */ 165 if (cluster_info->root_domain) { 166 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root; 167 int i; 168 169 for (i = 0; i < uncore_root->power_domain_count; ++i) { 170 int j; 171 172 for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) 173 write_control_freq(&uncore_root->pd_info[i].cluster_infos[j], 174 input, min_max); 175 } 176 177 if (min_max) 178 uncore_root->max_ratio = input; 179 else 180 uncore_root->min_ratio = input; 181 182 return 0; 183 } 184 185 if (min_max && uncore_root->max_ratio && uncore_root->max_ratio < input) 186 return -EINVAL; 187 188 if (!min_max && uncore_root->min_ratio && uncore_root->min_ratio > input) 189 return -EINVAL; 190 191 write_control_freq(cluster_info, input, min_max); 192 193 return 0; 194} 195 196/* Callback for sysfs read for the current uncore frequency. Called under mutex locks */ 197static int uncore_read_freq(struct uncore_data *data, unsigned int *freq) 198{ 199 struct tpmi_uncore_cluster_info *cluster_info; 200 u64 status; 201 202 cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data); 203 if (cluster_info->root_domain) 204 return -ENODATA; 205 206 status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX); 207 *freq = FIELD_GET(UNCORE_GENMASK_CURRENT_RATIO, status) * UNCORE_FREQ_KHZ_MULTIPLIER; 208 209 return 0; 210} 211 212static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore) 213{ 214 int i; 215 216 for (i = 0; i < tpmi_uncore->power_domain_count; ++i) { 217 struct tpmi_uncore_power_domain_info *pd_info; 218 int j; 219 220 pd_info = &tpmi_uncore->pd_info[i]; 221 if (!pd_info->uncore_base) 222 continue; 223 224 for (j = 0; j < pd_info->cluster_count; ++j) { 225 struct tpmi_uncore_cluster_info *cluster_info; 226 227 cluster_info = &pd_info->cluster_infos[j]; 228 uncore_freq_remove_die_entry(&cluster_info->uncore_data); 229 } 230 } 231} 232 233#define UNCORE_VERSION_MASK GENMASK_ULL(7, 0) 234#define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK GENMASK_ULL(15, 8) 235#define UNCORE_CLUSTER_OFF_MASK GENMASK_ULL(7, 0) 236#define UNCORE_MAX_CLUSTER_PER_DOMAIN 8 237 238static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id) 239{ 240 bool read_blocked = 0, write_blocked = 0; 241 struct intel_tpmi_plat_info *plat_info; 242 struct tpmi_uncore_struct *tpmi_uncore; 243 bool uncore_sysfs_added = false; 244 int ret, i, pkg = 0; 245 int num_resources; 246 247 ret = tpmi_get_feature_status(auxdev, TPMI_ID_UNCORE, &read_blocked, &write_blocked); 248 if (ret) 249 dev_info(&auxdev->dev, "Can't read feature status: ignoring blocked status\n"); 250 251 if (read_blocked) { 252 dev_info(&auxdev->dev, "Firmware has blocked reads, exiting\n"); 253 return -ENODEV; 254 } 255 256 /* Get number of power domains, which is equal to number of resources */ 257 num_resources = tpmi_get_resource_count(auxdev); 258 if (!num_resources) 259 return -EINVAL; 260 261 /* Register callbacks to uncore core */ 262 ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq, 263 uncore_read_freq); 264 if (ret) 265 return ret; 266 267 /* Allocate uncore instance per package */ 268 tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL); 269 if (!tpmi_uncore) { 270 ret = -ENOMEM; 271 goto err_rem_common; 272 } 273 274 /* Allocate memory for all power domains in a package */ 275 tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources, 276 sizeof(*tpmi_uncore->pd_info), 277 GFP_KERNEL); 278 if (!tpmi_uncore->pd_info) { 279 ret = -ENOMEM; 280 goto err_rem_common; 281 } 282 283 tpmi_uncore->power_domain_count = num_resources; 284 tpmi_uncore->write_blocked = write_blocked; 285 286 /* Get the package ID from the TPMI core */ 287 plat_info = tpmi_get_platform_data(auxdev); 288 if (plat_info) 289 pkg = plat_info->package_id; 290 else 291 dev_info(&auxdev->dev, "Platform information is NULL\n"); 292 293 for (i = 0; i < num_resources; ++i) { 294 struct tpmi_uncore_power_domain_info *pd_info; 295 struct resource *res; 296 u64 cluster_offset; 297 u8 cluster_mask; 298 int mask, j; 299 u64 header; 300 301 res = tpmi_get_resource_at_index(auxdev, i); 302 if (!res) 303 continue; 304 305 pd_info = &tpmi_uncore->pd_info[i]; 306 307 pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res); 308 if (IS_ERR(pd_info->uncore_base)) { 309 ret = PTR_ERR(pd_info->uncore_base); 310 /* 311 * Set to NULL so that clean up can still remove other 312 * entries already created if any by 313 * remove_cluster_entries() 314 */ 315 pd_info->uncore_base = NULL; 316 goto remove_clusters; 317 } 318 319 /* Check for version and skip this resource if there is mismatch */ 320 header = readq(pd_info->uncore_base); 321 pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK; 322 323 if (pd_info->ufs_header_ver == TPMI_VERSION_INVALID) 324 continue; 325 326 if (TPMI_MAJOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MAJOR_VERSION) { 327 dev_err(&auxdev->dev, "Uncore: Unsupported major version:%lx\n", 328 TPMI_MAJOR_VERSION(pd_info->ufs_header_ver)); 329 ret = -ENODEV; 330 goto remove_clusters; 331 } 332 333 if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) > UNCORE_MINOR_VERSION) 334 dev_info(&auxdev->dev, "Uncore: Ignore: Unsupported minor version:%lx\n", 335 TPMI_MINOR_VERSION(pd_info->ufs_header_ver)); 336 337 /* Get Cluster ID Mask */ 338 cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header); 339 if (!cluster_mask) { 340 dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask); 341 continue; 342 } 343 344 /* Find out number of clusters in this resource */ 345 pd_info->cluster_count = hweight8(cluster_mask); 346 347 pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count, 348 sizeof(struct tpmi_uncore_cluster_info), 349 GFP_KERNEL); 350 if (!pd_info->cluster_infos) { 351 ret = -ENOMEM; 352 goto remove_clusters; 353 } 354 /* 355 * Each byte in the register point to status and control 356 * registers belonging to cluster id 0-8. 357 */ 358 cluster_offset = readq(pd_info->uncore_base + 359 UNCORE_FABRIC_CLUSTER_OFFSET); 360 361 for (j = 0; j < pd_info->cluster_count; ++j) { 362 struct tpmi_uncore_cluster_info *cluster_info; 363 364 /* Get the offset for this cluster */ 365 mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK); 366 /* Offset in QWORD, so change to bytes */ 367 mask <<= 3; 368 369 cluster_info = &pd_info->cluster_infos[j]; 370 371 cluster_info->cluster_base = pd_info->uncore_base + mask; 372 373 cluster_info->uncore_data.package_id = pkg; 374 /* There are no dies like Cascade Lake */ 375 cluster_info->uncore_data.die_id = 0; 376 cluster_info->uncore_data.domain_id = i; 377 cluster_info->uncore_data.cluster_id = j; 378 379 cluster_info->uncore_root = tpmi_uncore; 380 381 ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0); 382 if (ret) { 383 cluster_info->cluster_base = NULL; 384 goto remove_clusters; 385 } 386 /* Point to next cluster offset */ 387 cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN; 388 uncore_sysfs_added = true; 389 } 390 } 391 392 if (!uncore_sysfs_added) { 393 ret = -ENODEV; 394 goto remove_clusters; 395 } 396 397 auxiliary_set_drvdata(auxdev, tpmi_uncore); 398 399 tpmi_uncore->root_cluster.root_domain = true; 400 tpmi_uncore->root_cluster.uncore_root = tpmi_uncore; 401 402 tpmi_uncore->root_cluster.uncore_data.package_id = pkg; 403 tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID; 404 ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0); 405 if (ret) 406 goto remove_clusters; 407 408 return 0; 409 410remove_clusters: 411 remove_cluster_entries(tpmi_uncore); 412err_rem_common: 413 uncore_freq_common_exit(); 414 415 return ret; 416} 417 418static void uncore_remove(struct auxiliary_device *auxdev) 419{ 420 struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev); 421 422 uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data); 423 remove_cluster_entries(tpmi_uncore); 424 425 uncore_freq_common_exit(); 426} 427 428static const struct auxiliary_device_id intel_uncore_id_table[] = { 429 { .name = "intel_vsec.tpmi-uncore" }, 430 {} 431}; 432MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table); 433 434static struct auxiliary_driver intel_uncore_aux_driver = { 435 .id_table = intel_uncore_id_table, 436 .remove = uncore_remove, 437 .probe = uncore_probe, 438}; 439 440module_auxiliary_driver(intel_uncore_aux_driver); 441 442MODULE_IMPORT_NS(INTEL_TPMI); 443MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY); 444MODULE_DESCRIPTION("Intel TPMI UFS Driver"); 445MODULE_LICENSE("GPL"); 446