1// SPDX-License-Identifier: GPL-2.0 2// Copyright (c) 2017 Cadence 3// Cadence PCIe host controller driver. 4// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> 5 6#include <linux/delay.h> 7#include <linux/kernel.h> 8#include <linux/list_sort.h> 9#include <linux/of_address.h> 10#include <linux/of_pci.h> 11#include <linux/platform_device.h> 12 13#include "pcie-cadence.h" 14 15#define LINK_RETRAIN_TIMEOUT HZ 16 17static u64 bar_max_size[] = { 18 [RP_BAR0] = _ULL(128 * SZ_2G), 19 [RP_BAR1] = SZ_2G, 20 [RP_NO_BAR] = _BITULL(63), 21}; 22 23static u8 bar_aperture_mask[] = { 24 [RP_BAR0] = 0x1F, 25 [RP_BAR1] = 0xF, 26}; 27 28void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, 29 int where) 30{ 31 struct pci_host_bridge *bridge = pci_find_host_bridge(bus); 32 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge); 33 struct cdns_pcie *pcie = &rc->pcie; 34 unsigned int busn = bus->number; 35 u32 addr0, desc0; 36 37 if (pci_is_root_bus(bus)) { 38 /* 39 * Only the root port (devfn == 0) is connected to this bus. 40 * All other PCI devices are behind some bridge hence on another 41 * bus. 42 */ 43 if (devfn) 44 return NULL; 45 46 return pcie->reg_base + (where & 0xfff); 47 } 48 /* Check that the link is up */ 49 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1)) 50 return NULL; 51 /* Clear AXI link-down status */ 52 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0); 53 54 /* Update Output registers for AXI region 0. */ 55 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) | 56 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) | 57 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn); 58 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0); 59 60 /* Configuration Type 0 or Type 1 access. */ 61 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID | 62 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0); 63 /* 64 * The bus number was already set once for all in desc1 by 65 * cdns_pcie_host_init_address_translation(). 66 */ 67 if (busn == bridge->busnr + 1) 68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0; 69 else 70 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1; 71 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0); 72 73 return rc->cfg_base + (where & 0xfff); 74} 75 76static struct pci_ops cdns_pcie_host_ops = { 77 .map_bus = cdns_pci_map_bus, 78 .read = pci_generic_config_read, 79 .write = pci_generic_config_write, 80}; 81 82static int cdns_pcie_host_training_complete(struct cdns_pcie *pcie) 83{ 84 u32 pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; 85 unsigned long end_jiffies; 86 u16 lnk_stat; 87 88 /* Wait for link training to complete. Exit after timeout. */ 89 end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT; 90 do { 91 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); 92 if (!(lnk_stat & PCI_EXP_LNKSTA_LT)) 93 break; 94 usleep_range(0, 1000); 95 } while (time_before(jiffies, end_jiffies)); 96 97 if (!(lnk_stat & PCI_EXP_LNKSTA_LT)) 98 return 0; 99 100 return -ETIMEDOUT; 101} 102 103static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie) 104{ 105 struct device *dev = pcie->dev; 106 int retries; 107 108 /* Check if the link is up or not */ 109 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) { 110 if (cdns_pcie_link_up(pcie)) { 111 dev_info(dev, "Link up\n"); 112 return 0; 113 } 114 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); 115 } 116 117 return -ETIMEDOUT; 118} 119 120static int cdns_pcie_retrain(struct cdns_pcie *pcie) 121{ 122 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; 123 u16 lnk_stat, lnk_ctl; 124 int ret = 0; 125 126 /* 127 * Set retrain bit if current speed is 2.5 GB/s, 128 * but the PCIe root port support is > 2.5 GB/s. 129 */ 130 131 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off + 132 PCI_EXP_LNKCAP)); 133 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB) 134 return ret; 135 136 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); 137 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) { 138 lnk_ctl = cdns_pcie_rp_readw(pcie, 139 pcie_cap_off + PCI_EXP_LNKCTL); 140 lnk_ctl |= PCI_EXP_LNKCTL_RL; 141 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL, 142 lnk_ctl); 143 144 ret = cdns_pcie_host_training_complete(pcie); 145 if (ret) 146 return ret; 147 148 ret = cdns_pcie_host_wait_for_link(pcie); 149 } 150 return ret; 151} 152 153static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie) 154{ 155 u32 val; 156 157 val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL); 158 cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN); 159} 160 161static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc) 162{ 163 struct cdns_pcie *pcie = &rc->pcie; 164 int ret; 165 166 ret = cdns_pcie_host_wait_for_link(pcie); 167 168 /* 169 * Retrain link for Gen2 training defect 170 * if quirk flag is set. 171 */ 172 if (!ret && rc->quirk_retrain_flag) 173 ret = cdns_pcie_retrain(pcie); 174 175 return ret; 176} 177 178static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc) 179{ 180 struct cdns_pcie *pcie = &rc->pcie; 181 u32 value, ctrl; 182 u32 id; 183 184 /* 185 * Set the root complex BAR configuration register: 186 * - disable both BAR0 and BAR1. 187 * - enable Prefetchable Memory Base and Limit registers in type 1 188 * config space (64 bits). 189 * - enable IO Base and Limit registers in type 1 config 190 * space (32 bits). 191 */ 192 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; 193 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) | 194 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) | 195 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE | 196 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS | 197 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE | 198 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS; 199 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); 200 201 /* Set root port configuration space */ 202 if (rc->vendor_id != 0xffff) { 203 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) | 204 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id); 205 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id); 206 } 207 208 if (rc->device_id != 0xffff) 209 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id); 210 211 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0); 212 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0); 213 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); 214 215 return 0; 216} 217 218static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc, 219 enum cdns_pcie_rp_bar bar, 220 u64 cpu_addr, u64 size, 221 unsigned long flags) 222{ 223 struct cdns_pcie *pcie = &rc->pcie; 224 u32 addr0, addr1, aperture, value; 225 226 if (!rc->avail_ib_bar[bar]) 227 return -EBUSY; 228 229 rc->avail_ib_bar[bar] = false; 230 231 aperture = ilog2(size); 232 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) | 233 (lower_32_bits(cpu_addr) & GENMASK(31, 8)); 234 addr1 = upper_32_bits(cpu_addr); 235 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0); 236 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1); 237 238 if (bar == RP_NO_BAR) 239 return 0; 240 241 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG); 242 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) | 243 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) | 244 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) | 245 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) | 246 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2)); 247 if (size + cpu_addr >= SZ_4G) { 248 if (!(flags & IORESOURCE_PREFETCH)) 249 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar); 250 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar); 251 } else { 252 if (!(flags & IORESOURCE_PREFETCH)) 253 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar); 254 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar); 255 } 256 257 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture); 258 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); 259 260 return 0; 261} 262 263static enum cdns_pcie_rp_bar 264cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size) 265{ 266 enum cdns_pcie_rp_bar bar, sel_bar; 267 268 sel_bar = RP_BAR_UNDEFINED; 269 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { 270 if (!rc->avail_ib_bar[bar]) 271 continue; 272 273 if (size <= bar_max_size[bar]) { 274 if (sel_bar == RP_BAR_UNDEFINED) { 275 sel_bar = bar; 276 continue; 277 } 278 279 if (bar_max_size[bar] < bar_max_size[sel_bar]) 280 sel_bar = bar; 281 } 282 } 283 284 return sel_bar; 285} 286 287static enum cdns_pcie_rp_bar 288cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size) 289{ 290 enum cdns_pcie_rp_bar bar, sel_bar; 291 292 sel_bar = RP_BAR_UNDEFINED; 293 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { 294 if (!rc->avail_ib_bar[bar]) 295 continue; 296 297 if (size >= bar_max_size[bar]) { 298 if (sel_bar == RP_BAR_UNDEFINED) { 299 sel_bar = bar; 300 continue; 301 } 302 303 if (bar_max_size[bar] > bar_max_size[sel_bar]) 304 sel_bar = bar; 305 } 306 } 307 308 return sel_bar; 309} 310 311static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc, 312 struct resource_entry *entry) 313{ 314 u64 cpu_addr, pci_addr, size, winsize; 315 struct cdns_pcie *pcie = &rc->pcie; 316 struct device *dev = pcie->dev; 317 enum cdns_pcie_rp_bar bar; 318 unsigned long flags; 319 int ret; 320 321 cpu_addr = entry->res->start; 322 pci_addr = entry->res->start - entry->offset; 323 flags = entry->res->flags; 324 size = resource_size(entry->res); 325 326 if (entry->offset) { 327 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n", 328 pci_addr, cpu_addr); 329 return -EINVAL; 330 } 331 332 while (size > 0) { 333 /* 334 * Try to find a minimum BAR whose size is greater than 335 * or equal to the remaining resource_entry size. This will 336 * fail if the size of each of the available BARs is less than 337 * the remaining resource_entry size. 338 * If a minimum BAR is found, IB ATU will be configured and 339 * exited. 340 */ 341 bar = cdns_pcie_host_find_min_bar(rc, size); 342 if (bar != RP_BAR_UNDEFINED) { 343 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, 344 size, flags); 345 if (ret) 346 dev_err(dev, "IB BAR: %d config failed\n", bar); 347 return ret; 348 } 349 350 /* 351 * If the control reaches here, it would mean the remaining 352 * resource_entry size cannot be fitted in a single BAR. So we 353 * find a maximum BAR whose size is less than or equal to the 354 * remaining resource_entry size and split the resource entry 355 * so that part of resource entry is fitted inside the maximum 356 * BAR. The remaining size would be fitted during the next 357 * iteration of the loop. 358 * If a maximum BAR is not found, there is no way we can fit 359 * this resource_entry, so we error out. 360 */ 361 bar = cdns_pcie_host_find_max_bar(rc, size); 362 if (bar == RP_BAR_UNDEFINED) { 363 dev_err(dev, "No free BAR to map cpu_addr %llx\n", 364 cpu_addr); 365 return -EINVAL; 366 } 367 368 winsize = bar_max_size[bar]; 369 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize, 370 flags); 371 if (ret) { 372 dev_err(dev, "IB BAR: %d config failed\n", bar); 373 return ret; 374 } 375 376 size -= winsize; 377 cpu_addr += winsize; 378 } 379 380 return 0; 381} 382 383static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a, 384 const struct list_head *b) 385{ 386 struct resource_entry *entry1, *entry2; 387 388 entry1 = container_of(a, struct resource_entry, node); 389 entry2 = container_of(b, struct resource_entry, node); 390 391 return resource_size(entry2->res) - resource_size(entry1->res); 392} 393 394static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc) 395{ 396 struct cdns_pcie *pcie = &rc->pcie; 397 struct device *dev = pcie->dev; 398 struct device_node *np = dev->of_node; 399 struct pci_host_bridge *bridge; 400 struct resource_entry *entry; 401 u32 no_bar_nbits = 32; 402 int err; 403 404 bridge = pci_host_bridge_from_priv(rc); 405 if (!bridge) 406 return -ENOMEM; 407 408 if (list_empty(&bridge->dma_ranges)) { 409 of_property_read_u32(np, "cdns,no-bar-match-nbits", 410 &no_bar_nbits); 411 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0, 412 (u64)1 << no_bar_nbits, 0); 413 if (err) 414 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR); 415 return err; 416 } 417 418 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp); 419 420 resource_list_for_each_entry(entry, &bridge->dma_ranges) { 421 err = cdns_pcie_host_bar_config(rc, entry); 422 if (err) { 423 dev_err(dev, "Fail to configure IB using dma-ranges\n"); 424 return err; 425 } 426 } 427 428 return 0; 429} 430 431static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) 432{ 433 struct cdns_pcie *pcie = &rc->pcie; 434 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc); 435 struct resource *cfg_res = rc->cfg_res; 436 struct resource_entry *entry; 437 u64 cpu_addr = cfg_res->start; 438 u32 addr0, addr1, desc1; 439 int r, busnr = 0; 440 441 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); 442 if (entry) 443 busnr = entry->res->start; 444 445 /* 446 * Reserve region 0 for PCI configure space accesses: 447 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by 448 * cdns_pci_map_bus(), other region registers are set here once for all. 449 */ 450 addr1 = 0; /* Should be programmed to zero. */ 451 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr); 452 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1); 453 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1); 454 455 if (pcie->ops->cpu_addr_fixup) 456 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); 457 458 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) | 459 (lower_32_bits(cpu_addr) & GENMASK(31, 8)); 460 addr1 = upper_32_bits(cpu_addr); 461 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0); 462 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1); 463 464 r = 1; 465 resource_list_for_each_entry(entry, &bridge->windows) { 466 struct resource *res = entry->res; 467 u64 pci_addr = res->start - entry->offset; 468 469 if (resource_type(res) == IORESOURCE_IO) 470 cdns_pcie_set_outbound_region(pcie, busnr, 0, r, 471 true, 472 pci_pio_to_address(res->start), 473 pci_addr, 474 resource_size(res)); 475 else 476 cdns_pcie_set_outbound_region(pcie, busnr, 0, r, 477 false, 478 res->start, 479 pci_addr, 480 resource_size(res)); 481 482 r++; 483 } 484 485 return cdns_pcie_host_map_dma_ranges(rc); 486} 487 488static int cdns_pcie_host_init(struct device *dev, 489 struct cdns_pcie_rc *rc) 490{ 491 int err; 492 493 err = cdns_pcie_host_init_root_port(rc); 494 if (err) 495 return err; 496 497 return cdns_pcie_host_init_address_translation(rc); 498} 499 500int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) 501{ 502 struct device *dev = rc->pcie.dev; 503 struct platform_device *pdev = to_platform_device(dev); 504 struct device_node *np = dev->of_node; 505 struct pci_host_bridge *bridge; 506 enum cdns_pcie_rp_bar bar; 507 struct cdns_pcie *pcie; 508 struct resource *res; 509 int ret; 510 511 bridge = pci_host_bridge_from_priv(rc); 512 if (!bridge) 513 return -ENOMEM; 514 515 pcie = &rc->pcie; 516 pcie->is_rc = true; 517 518 rc->vendor_id = 0xffff; 519 of_property_read_u32(np, "vendor-id", &rc->vendor_id); 520 521 rc->device_id = 0xffff; 522 of_property_read_u32(np, "device-id", &rc->device_id); 523 524 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg"); 525 if (IS_ERR(pcie->reg_base)) { 526 dev_err(dev, "missing \"reg\"\n"); 527 return PTR_ERR(pcie->reg_base); 528 } 529 530 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); 531 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res); 532 if (IS_ERR(rc->cfg_base)) 533 return PTR_ERR(rc->cfg_base); 534 rc->cfg_res = res; 535 536 if (rc->quirk_detect_quiet_flag) 537 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie); 538 539 cdns_pcie_host_enable_ptm_response(pcie); 540 541 ret = cdns_pcie_start_link(pcie); 542 if (ret) { 543 dev_err(dev, "Failed to start link\n"); 544 return ret; 545 } 546 547 ret = cdns_pcie_host_start_link(rc); 548 if (ret) 549 dev_dbg(dev, "PCIe link never came up\n"); 550 551 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) 552 rc->avail_ib_bar[bar] = true; 553 554 ret = cdns_pcie_host_init(dev, rc); 555 if (ret) 556 return ret; 557 558 if (!bridge->ops) 559 bridge->ops = &cdns_pcie_host_ops; 560 561 ret = pci_host_probe(bridge); 562 if (ret < 0) 563 goto err_init; 564 565 return 0; 566 567 err_init: 568 pm_runtime_put_sync(dev); 569 570 return ret; 571} 572