1201052Smarius/*- 2201052Smarius * Copyright (c) 1999, 2000 Matthew R. Green 3201052Smarius * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org> 4201052Smarius * Copyright (c) 2009 by Marius Strobl <marius@FreeBSD.org> 5201052Smarius * All rights reserved. 6201052Smarius * 7201052Smarius * Redistribution and use in source and binary forms, with or without 8201052Smarius * modification, are permitted provided that the following conditions 9201052Smarius * are met: 10201052Smarius * 1. Redistributions of source code must retain the above copyright 11201052Smarius * notice, this list of conditions and the following disclaimer. 12201052Smarius * 2. Redistributions in binary form must reproduce the above copyright 13201052Smarius * notice, this list of conditions and the following disclaimer in the 14201052Smarius * documentation and/or other materials provided with the distribution. 15201052Smarius * 3. The name of the author may not be used to endorse or promote products 16201052Smarius * derived from this software without specific prior written permission. 17201052Smarius * 18201052Smarius * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19201052Smarius * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20201052Smarius * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21201052Smarius * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22201052Smarius * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23201052Smarius * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24201052Smarius * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25201052Smarius * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26201052Smarius * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27201052Smarius * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28201052Smarius * SUCH DAMAGE. 29201052Smarius * 30201052Smarius * from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp 31201052Smarius * from: FreeBSD: psycho.c 183152 2008-09-18 19:45:22Z marius 32201052Smarius */ 33201052Smarius 34201052Smarius#include <sys/cdefs.h> 35201052Smarius__FBSDID("$FreeBSD$"); 36201052Smarius 37201052Smarius/* 38201052Smarius * Driver for `Fire' JBus to PCI Express and `Oberon' Uranus to PCI Express 39201052Smarius * bridges 40201052Smarius */ 41201052Smarius 42201052Smarius#include "opt_fire.h" 43201052Smarius#include "opt_ofw_pci.h" 44201052Smarius 45201052Smarius#include <sys/param.h> 46201052Smarius#include <sys/systm.h> 47201052Smarius#include <sys/bus.h> 48201052Smarius#include <sys/interrupt.h> 49201052Smarius#include <sys/kernel.h> 50201052Smarius#include <sys/lock.h> 51201052Smarius#include <sys/malloc.h> 52201052Smarius#include <sys/module.h> 53201052Smarius#include <sys/mutex.h> 54201052Smarius#include <sys/pciio.h> 55201052Smarius#include <sys/pcpu.h> 56201052Smarius#include <sys/rman.h> 57201052Smarius#include <sys/smp.h> 58201052Smarius#include <sys/sysctl.h> 59201052Smarius#include <sys/timetc.h> 60201052Smarius 61201052Smarius#include <dev/ofw/ofw_bus.h> 62201052Smarius#include <dev/ofw/ofw_pci.h> 63201052Smarius#include <dev/ofw/openfirm.h> 64201052Smarius 65201052Smarius#include <vm/vm.h> 66201052Smarius#include <vm/pmap.h> 67201052Smarius 68201052Smarius#include <machine/bus.h> 69201052Smarius#include <machine/bus_common.h> 70201052Smarius#include <machine/bus_private.h> 71201052Smarius#include <machine/fsr.h> 72201052Smarius#include <machine/iommureg.h> 73201052Smarius#include <machine/iommuvar.h> 74201052Smarius#include <machine/pmap.h> 75201052Smarius#include <machine/resource.h> 76201052Smarius 77201052Smarius#include <dev/pci/pcireg.h> 78201052Smarius#include <dev/pci/pcivar.h> 79201052Smarius 80201052Smarius#include <sparc64/pci/ofw_pci.h> 81201052Smarius#include <sparc64/pci/firereg.h> 82201052Smarius#include <sparc64/pci/firevar.h> 83201052Smarius 84201052Smarius#include "pcib_if.h" 85201052Smarius 86202003Smariusstruct fire_msiqarg; 87202003Smarius 88201052Smariusstatic const struct fire_desc *fire_get_desc(device_t dev); 89201052Smariusstatic void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map, 90201052Smarius bus_dmasync_op_t op); 91201052Smariusstatic int fire_get_intrmap(struct fire_softc *sc, u_int ino, 92201052Smarius bus_addr_t *intrmapptr, bus_addr_t *intrclrptr); 93201052Smariusstatic void fire_intr_assign(void *arg); 94201052Smariusstatic void fire_intr_clear(void *arg); 95201052Smariusstatic void fire_intr_disable(void *arg); 96201052Smariusstatic void fire_intr_enable(void *arg); 97201052Smariusstatic int fire_intr_register(struct fire_softc *sc, u_int ino); 98202003Smariusstatic inline void fire_msiq_common(struct intr_vector *iv, 99202003Smarius struct fire_msiqarg *fmqa); 100202003Smariusstatic void fire_msiq_filter(void *cookie); 101201052Smariusstatic void fire_msiq_handler(void *cookie); 102201052Smariusstatic void fire_set_intr(struct fire_softc *sc, u_int index, u_int ino, 103201052Smarius driver_filter_t handler, void *arg); 104201052Smariusstatic timecounter_get_t fire_get_timecount; 105201052Smarius 106201052Smarius/* Interrupt handlers */ 107201052Smariusstatic driver_filter_t fire_dmc_pec; 108201052Smariusstatic driver_filter_t fire_pcie; 109201052Smariusstatic driver_filter_t fire_xcb; 110201052Smarius 111201052Smarius/* 112201052Smarius * Methods 113201052Smarius */ 114201052Smariusstatic bus_activate_resource_t fire_activate_resource; 115225931Smariusstatic bus_adjust_resource_t fire_adjust_resource; 116201052Smariusstatic pcib_alloc_msi_t fire_alloc_msi; 117201052Smariusstatic pcib_alloc_msix_t fire_alloc_msix; 118201052Smariusstatic bus_alloc_resource_t fire_alloc_resource; 119201052Smariusstatic device_attach_t fire_attach; 120201052Smariusstatic bus_get_dma_tag_t fire_get_dma_tag; 121201052Smariusstatic ofw_bus_get_node_t fire_get_node; 122201052Smariusstatic pcib_map_msi_t fire_map_msi; 123201052Smariusstatic pcib_maxslots_t fire_maxslots; 124201052Smariusstatic device_probe_t fire_probe; 125201052Smariusstatic pcib_read_config_t fire_read_config; 126201052Smariusstatic bus_read_ivar_t fire_read_ivar; 127201052Smariusstatic pcib_release_msi_t fire_release_msi; 128201052Smariusstatic pcib_release_msix_t fire_release_msix; 129201052Smariusstatic pcib_route_interrupt_t fire_route_interrupt; 130201052Smariusstatic bus_setup_intr_t fire_setup_intr; 131201052Smariusstatic bus_teardown_intr_t fire_teardown_intr; 132201052Smariusstatic pcib_write_config_t fire_write_config; 133201052Smarius 134201052Smariusstatic device_method_t fire_methods[] = { 135201052Smarius /* Device interface */ 136201052Smarius DEVMETHOD(device_probe, fire_probe), 137201052Smarius DEVMETHOD(device_attach, fire_attach), 138201052Smarius DEVMETHOD(device_shutdown, bus_generic_shutdown), 139201052Smarius DEVMETHOD(device_suspend, bus_generic_suspend), 140201052Smarius DEVMETHOD(device_resume, bus_generic_resume), 141201052Smarius 142201052Smarius /* Bus interface */ 143201052Smarius DEVMETHOD(bus_read_ivar, fire_read_ivar), 144201052Smarius DEVMETHOD(bus_setup_intr, fire_setup_intr), 145201052Smarius DEVMETHOD(bus_teardown_intr, fire_teardown_intr), 146201052Smarius DEVMETHOD(bus_alloc_resource, fire_alloc_resource), 147225931Smarius DEVMETHOD(bus_activate_resource, fire_activate_resource), 148225931Smarius DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), 149225931Smarius DEVMETHOD(bus_adjust_resource, fire_adjust_resource), 150225931Smarius DEVMETHOD(bus_release_resource, bus_generic_release_resource), 151201052Smarius DEVMETHOD(bus_get_dma_tag, fire_get_dma_tag), 152201052Smarius 153201052Smarius /* pcib interface */ 154201052Smarius DEVMETHOD(pcib_maxslots, fire_maxslots), 155201052Smarius DEVMETHOD(pcib_read_config, fire_read_config), 156201052Smarius DEVMETHOD(pcib_write_config, fire_write_config), 157201052Smarius DEVMETHOD(pcib_route_interrupt, fire_route_interrupt), 158201052Smarius DEVMETHOD(pcib_alloc_msi, fire_alloc_msi), 159201052Smarius DEVMETHOD(pcib_release_msi, fire_release_msi), 160201052Smarius DEVMETHOD(pcib_alloc_msix, fire_alloc_msix), 161201052Smarius DEVMETHOD(pcib_release_msix, fire_release_msix), 162201052Smarius DEVMETHOD(pcib_map_msi, fire_map_msi), 163201052Smarius 164201052Smarius /* ofw_bus interface */ 165201052Smarius DEVMETHOD(ofw_bus_get_node, fire_get_node), 166201052Smarius 167227843Smarius DEVMETHOD_END 168201052Smarius}; 169201052Smarius 170201052Smariusstatic devclass_t fire_devclass; 171201052Smarius 172201052SmariusDEFINE_CLASS_0(pcib, fire_driver, fire_methods, sizeof(struct fire_softc)); 173201052SmariusEARLY_DRIVER_MODULE(fire, nexus, fire_driver, fire_devclass, 0, 0, 174201052Smarius BUS_PASS_BUS); 175201052SmariusMODULE_DEPEND(fire, nexus, 1, 1, 1); 176201052Smarius 177201052Smariusstatic const struct intr_controller fire_ic = { 178201052Smarius fire_intr_enable, 179201052Smarius fire_intr_disable, 180201052Smarius fire_intr_assign, 181201052Smarius fire_intr_clear 182201052Smarius}; 183201052Smarius 184201052Smariusstruct fire_icarg { 185201052Smarius struct fire_softc *fica_sc; 186201052Smarius bus_addr_t fica_map; 187201052Smarius bus_addr_t fica_clr; 188201052Smarius}; 189201052Smarius 190202003Smariusstatic const struct intr_controller fire_msiqc_filter = { 191202003Smarius fire_intr_enable, 192202003Smarius fire_intr_disable, 193202003Smarius fire_intr_assign, 194202003Smarius NULL 195202003Smarius}; 196202003Smarius 197201052Smariusstruct fire_msiqarg { 198201052Smarius struct fire_icarg fmqa_fica; 199201052Smarius struct mtx fmqa_mtx; 200201052Smarius struct fo_msiq_record *fmqa_base; 201201052Smarius uint64_t fmqa_head; 202201052Smarius uint64_t fmqa_tail; 203201052Smarius uint32_t fmqa_msiq; 204201052Smarius uint32_t fmqa_msi; 205201052Smarius}; 206201052Smarius 207201052Smarius#define FIRE_PERF_CNT_QLTY 100 208201052Smarius 209201052Smarius#define FIRE_SPC_BARRIER(spc, sc, offs, len, flags) \ 210201052Smarius bus_barrier((sc)->sc_mem_res[(spc)], (offs), (len), (flags)) 211201052Smarius#define FIRE_SPC_READ_8(spc, sc, offs) \ 212201052Smarius bus_read_8((sc)->sc_mem_res[(spc)], (offs)) 213201052Smarius#define FIRE_SPC_WRITE_8(spc, sc, offs, v) \ 214201052Smarius bus_write_8((sc)->sc_mem_res[(spc)], (offs), (v)) 215201052Smarius 216201052Smarius#ifndef FIRE_DEBUG 217201052Smarius#define FIRE_SPC_SET(spc, sc, offs, reg, v) \ 218201052Smarius FIRE_SPC_WRITE_8((spc), (sc), (offs), (v)) 219201052Smarius#else 220201052Smarius#define FIRE_SPC_SET(spc, sc, offs, reg, v) do { \ 221201052Smarius device_printf((sc)->sc_dev, reg " 0x%016llx -> 0x%016llx\n", \ 222201052Smarius (unsigned long long)FIRE_SPC_READ_8((spc), (sc), (offs)), \ 223201052Smarius (unsigned long long)(v)); \ 224201052Smarius FIRE_SPC_WRITE_8((spc), (sc), (offs), (v)); \ 225201052Smarius } while (0) 226201052Smarius#endif 227201052Smarius 228201052Smarius#define FIRE_PCI_BARRIER(sc, offs, len, flags) \ 229201052Smarius FIRE_SPC_BARRIER(FIRE_PCI, (sc), (offs), len, flags) 230201052Smarius#define FIRE_PCI_READ_8(sc, offs) \ 231201052Smarius FIRE_SPC_READ_8(FIRE_PCI, (sc), (offs)) 232201052Smarius#define FIRE_PCI_WRITE_8(sc, offs, v) \ 233201052Smarius FIRE_SPC_WRITE_8(FIRE_PCI, (sc), (offs), (v)) 234201052Smarius#define FIRE_CTRL_BARRIER(sc, offs, len, flags) \ 235201052Smarius FIRE_SPC_BARRIER(FIRE_CTRL, (sc), (offs), len, flags) 236201052Smarius#define FIRE_CTRL_READ_8(sc, offs) \ 237201052Smarius FIRE_SPC_READ_8(FIRE_CTRL, (sc), (offs)) 238201052Smarius#define FIRE_CTRL_WRITE_8(sc, offs, v) \ 239201052Smarius FIRE_SPC_WRITE_8(FIRE_CTRL, (sc), (offs), (v)) 240201052Smarius 241201052Smarius#define FIRE_PCI_SET(sc, offs, v) \ 242201052Smarius FIRE_SPC_SET(FIRE_PCI, (sc), (offs), # offs, (v)) 243201052Smarius#define FIRE_CTRL_SET(sc, offs, v) \ 244201052Smarius FIRE_SPC_SET(FIRE_CTRL, (sc), (offs), # offs, (v)) 245201052Smarius 246201052Smariusstruct fire_desc { 247201052Smarius const char *fd_string; 248201052Smarius int fd_mode; 249201052Smarius const char *fd_name; 250201052Smarius}; 251201052Smarius 252242625Sdimstatic const struct fire_desc fire_compats[] = { 253201052Smarius { "pciex108e,80f0", FIRE_MODE_FIRE, "Fire" }, 254201052Smarius#if 0 255201052Smarius { "pciex108e,80f8", FIRE_MODE_OBERON, "Oberon" }, 256201052Smarius#endif 257201052Smarius { NULL, 0, NULL } 258201052Smarius}; 259201052Smarius 260201052Smariusstatic const struct fire_desc * 261201052Smariusfire_get_desc(device_t dev) 262201052Smarius{ 263201052Smarius const struct fire_desc *desc; 264201052Smarius const char *compat; 265201052Smarius 266201052Smarius compat = ofw_bus_get_compat(dev); 267201052Smarius if (compat == NULL) 268201052Smarius return (NULL); 269201052Smarius for (desc = fire_compats; desc->fd_string != NULL; desc++) 270201052Smarius if (strcmp(desc->fd_string, compat) == 0) 271201052Smarius return (desc); 272201052Smarius return (NULL); 273201052Smarius} 274201052Smarius 275201052Smariusstatic int 276201052Smariusfire_probe(device_t dev) 277201052Smarius{ 278201052Smarius const char *dtype; 279201052Smarius 280201052Smarius dtype = ofw_bus_get_type(dev); 281201052Smarius if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCIE) == 0 && 282201052Smarius fire_get_desc(dev) != NULL) { 283201052Smarius device_set_desc(dev, "Sun Host-PCIe bridge"); 284201052Smarius return (BUS_PROBE_GENERIC); 285201052Smarius } 286201052Smarius return (ENXIO); 287201052Smarius} 288201052Smarius 289201052Smariusstatic int 290201052Smariusfire_attach(device_t dev) 291201052Smarius{ 292201052Smarius struct fire_softc *sc; 293201052Smarius const struct fire_desc *desc; 294201052Smarius struct ofw_pci_msi_ranges msi_ranges; 295201052Smarius struct ofw_pci_msi_addr_ranges msi_addr_ranges; 296201052Smarius struct ofw_pci_msi_eq_to_devino msi_eq_to_devino; 297201052Smarius struct fire_msiqarg *fmqa; 298201052Smarius struct timecounter *tc; 299201052Smarius struct ofw_pci_ranges *range; 300201052Smarius uint64_t ino_bitmap, val; 301201052Smarius phandle_t node; 302201052Smarius uint32_t prop, prop_array[2]; 303201200Smarius int i, j, mode; 304201052Smarius u_int lw; 305201052Smarius uint16_t mps; 306201052Smarius 307201052Smarius sc = device_get_softc(dev); 308201052Smarius node = ofw_bus_get_node(dev); 309201052Smarius desc = fire_get_desc(dev); 310201052Smarius mode = desc->fd_mode; 311201052Smarius 312201052Smarius sc->sc_dev = dev; 313201052Smarius sc->sc_node = node; 314201052Smarius sc->sc_mode = mode; 315201052Smarius sc->sc_flags = 0; 316201052Smarius 317201052Smarius mtx_init(&sc->sc_msi_mtx, "msi_mtx", NULL, MTX_DEF); 318201052Smarius mtx_init(&sc->sc_pcib_mtx, "pcib_mtx", NULL, MTX_SPIN); 319201052Smarius 320201052Smarius /* 321201052Smarius * Fire and Oberon have two register banks: 322201052Smarius * (0) per-PBM PCI Express configuration and status registers 323201052Smarius * (1) (shared) Fire/Oberon controller configuration and status 324201052Smarius * registers 325201052Smarius */ 326201052Smarius for (i = 0; i < FIRE_NREG; i++) { 327201052Smarius j = i; 328201052Smarius sc->sc_mem_res[i] = bus_alloc_resource_any(dev, 329201052Smarius SYS_RES_MEMORY, &j, RF_ACTIVE); 330201052Smarius if (sc->sc_mem_res[i] == NULL) 331201052Smarius panic("%s: could not allocate register bank %d", 332201052Smarius __func__, i); 333201052Smarius } 334201052Smarius 335201052Smarius if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1) 336201052Smarius panic("%s: could not determine IGN", __func__); 337201052Smarius if (OF_getprop(node, "module-revision#", &prop, sizeof(prop)) == -1) 338219785Smarius panic("%s: could not determine module-revision", __func__); 339201052Smarius 340201052Smarius device_printf(dev, "%s, module-revision %d, IGN %#x\n", 341201052Smarius desc->fd_name, prop, sc->sc_ign); 342201052Smarius 343201052Smarius /* 344201052Smarius * Hunt through all the interrupt mapping regs and register 345201052Smarius * the interrupt controller for our interrupt vectors. We do 346201052Smarius * this early in order to be able to catch stray interrupts. 347201052Smarius */ 348201052Smarius i = OF_getprop(node, "ino-bitmap", (void *)prop_array, 349201052Smarius sizeof(prop_array)); 350201052Smarius if (i == -1) 351201052Smarius panic("%s: could not get ino-bitmap", __func__); 352201052Smarius ino_bitmap = ((uint64_t)prop_array[1] << 32) | prop_array[0]; 353201052Smarius for (i = 0; i <= FO_MAX_INO; i++) { 354201052Smarius if ((ino_bitmap & (1ULL << i)) == 0) 355201052Smarius continue; 356201052Smarius j = fire_intr_register(sc, i); 357201052Smarius if (j != 0) 358201052Smarius device_printf(dev, "could not register interrupt " 359201052Smarius "controller for INO %d (%d)\n", i, j); 360201052Smarius } 361201052Smarius 362201052Smarius /* JBC/UBC module initialization */ 363201052Smarius FIRE_CTRL_SET(sc, FO_XBC_ERR_LOG_EN, ~0ULL); 364201052Smarius FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); 365201052Smarius /* not enabled by OpenSolaris */ 366201052Smarius FIRE_CTRL_SET(sc, FO_XBC_INT_EN, ~0ULL); 367201052Smarius if (sc->sc_mode == FIRE_MODE_FIRE) { 368201052Smarius FIRE_CTRL_SET(sc, FIRE_JBUS_PAR_CTRL, 369201052Smarius FIRE_JBUS_PAR_CTRL_P_EN); 370201052Smarius FIRE_CTRL_SET(sc, FIRE_JBC_FATAL_RST_EN, 371201052Smarius ((1ULL << FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_SHFT) & 372201052Smarius FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_MASK) | 373201052Smarius FIRE_JBC_FATAL_RST_EN_MB_PEA_P_INT | 374201052Smarius FIRE_JBC_FATAL_RST_EN_CPE_P_INT | 375201052Smarius FIRE_JBC_FATAL_RST_EN_APE_P_INT | 376201052Smarius FIRE_JBC_FATAL_RST_EN_PIO_CPE_INT | 377201052Smarius FIRE_JBC_FATAL_RST_EN_JTCEEW_P_INT | 378201052Smarius FIRE_JBC_FATAL_RST_EN_JTCEEI_P_INT | 379201052Smarius FIRE_JBC_FATAL_RST_EN_JTCEER_P_INT); 380201052Smarius FIRE_CTRL_SET(sc, FIRE_JBC_CORE_BLOCK_INT_EN, ~0ULL); 381201052Smarius } 382201052Smarius 383201052Smarius /* TLU initialization */ 384201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_STAT_CLR, 385201052Smarius FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK); 386201052Smarius /* not enabled by OpenSolaris */ 387201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_INT_EN, 388201052Smarius FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK); 389201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_STAT_CLR, 390201052Smarius FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK); 391201052Smarius /* not enabled by OpenSolaris */ 392201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_INT_EN, 393201052Smarius FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK); 394201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_STAT_CLR, 395201052Smarius FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK); 396201052Smarius /* not enabled by OpenSolaris */ 397201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_INT_EN, 398201052Smarius FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK); 399201052Smarius val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) | 400201052Smarius ((FO_PCI_TLU_CTRL_L0S_TIM_DFLT << FO_PCI_TLU_CTRL_L0S_TIM_SHFT) & 401201052Smarius FO_PCI_TLU_CTRL_L0S_TIM_MASK) | 402201052Smarius ((FO_PCI_TLU_CTRL_CFG_DFLT << FO_PCI_TLU_CTRL_CFG_SHFT) & 403201052Smarius FO_PCI_TLU_CTRL_CFG_MASK); 404201052Smarius if (sc->sc_mode == FIRE_MODE_OBERON) 405201052Smarius val &= ~FO_PCI_TLU_CTRL_NWPR_EN; 406201052Smarius val |= FO_PCI_TLU_CTRL_CFG_REMAIN_DETECT_QUIET; 407201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_CTRL, val); 408201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_DEV_CTRL, 0); 409201052Smarius FIRE_PCI_SET(sc, FO_PCI_TLU_LNK_CTRL, FO_PCI_TLU_LNK_CTRL_CLK); 410201052Smarius 411201052Smarius /* DLU/LPU initialization */ 412201052Smarius if (sc->sc_mode == FIRE_MODE_OBERON) 413201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_INT_MASK, 0); 414201052Smarius else 415201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_RST, 0); 416201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_LNK_LYR_CFG, 417201052Smarius FO_PCI_LPU_LNK_LYR_CFG_VC0_EN); 418201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_FLW_CTRL_UPDT_CTRL, 419201052Smarius FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_NP_EN | 420201052Smarius FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_P_EN); 421201052Smarius if (sc->sc_mode == FIRE_MODE_OBERON) 422201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS, 423201052Smarius (OBERON_PCI_LPU_TXLNK_RPLY_TMR_THRS_DFLT << 424201052Smarius FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) & 425201052Smarius FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK); 426201052Smarius else { 427201052Smarius switch ((FIRE_PCI_READ_8(sc, FO_PCI_TLU_LNK_STAT) & 428201052Smarius FO_PCI_TLU_LNK_STAT_WDTH_MASK) >> 429201052Smarius FO_PCI_TLU_LNK_STAT_WDTH_SHFT) { 430201052Smarius case 1: 431201052Smarius lw = 0; 432201052Smarius break; 433201052Smarius case 4: 434201052Smarius lw = 1; 435201052Smarius break; 436201052Smarius case 8: 437201052Smarius lw = 2; 438201052Smarius break; 439201052Smarius case 16: 440201052Smarius lw = 3; 441201052Smarius break; 442201052Smarius default: 443201052Smarius lw = 0; 444201052Smarius } 445201052Smarius mps = (FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) & 446233701Smarius FO_PCI_TLU_CTRL_CFG_MPS_MASK) >> 447233701Smarius FO_PCI_TLU_CTRL_CFG_MPS_SHFT; 448201052Smarius i = sizeof(fire_freq_nak_tmr_thrs) / 449201052Smarius sizeof(*fire_freq_nak_tmr_thrs); 450233701Smarius if (mps >= i) 451201052Smarius mps = i - 1; 452201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS, 453201052Smarius (fire_freq_nak_tmr_thrs[mps][lw] << 454201052Smarius FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_SHFT) & 455201052Smarius FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_MASK); 456201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS, 457201052Smarius (fire_rply_tmr_thrs[mps][lw] << 458201052Smarius FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) & 459201052Smarius FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK); 460201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RTR_FIFO_PTR, 461201052Smarius ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_DFLT << 462201052Smarius FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_SHFT) & 463201052Smarius FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_MASK) | 464201052Smarius ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_DFLT << 465201052Smarius FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_SHFT) & 466201052Smarius FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_MASK)); 467201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG2, 468201052Smarius (FO_PCI_LPU_LTSSM_CFG2_12_TO_DFLT << 469201052Smarius FO_PCI_LPU_LTSSM_CFG2_12_TO_SHFT) & 470201052Smarius FO_PCI_LPU_LTSSM_CFG2_12_TO_MASK); 471201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG3, 472201052Smarius (FO_PCI_LPU_LTSSM_CFG3_2_TO_DFLT << 473201052Smarius FO_PCI_LPU_LTSSM_CFG3_2_TO_SHFT) & 474201052Smarius FO_PCI_LPU_LTSSM_CFG3_2_TO_MASK); 475201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG4, 476201052Smarius ((FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_DFLT << 477201052Smarius FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_SHFT) & 478201052Smarius FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_MASK) | 479201052Smarius ((FO_PCI_LPU_LTSSM_CFG4_N_FTS_DFLT << 480201052Smarius FO_PCI_LPU_LTSSM_CFG4_N_FTS_SHFT) & 481201052Smarius FO_PCI_LPU_LTSSM_CFG4_N_FTS_MASK)); 482201052Smarius FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG5, 0); 483201052Smarius } 484201052Smarius 485201052Smarius /* ILU initialization */ 486201052Smarius FIRE_PCI_SET(sc, FO_PCI_ILU_ERR_STAT_CLR, ~0ULL); 487201052Smarius /* not enabled by OpenSolaris */ 488201052Smarius FIRE_PCI_SET(sc, FO_PCI_ILU_INT_EN, ~0ULL); 489201052Smarius 490201052Smarius /* IMU initialization */ 491201052Smarius FIRE_PCI_SET(sc, FO_PCI_IMU_ERR_STAT_CLR, ~0ULL); 492201052Smarius FIRE_PCI_SET(sc, FO_PCI_IMU_INT_EN, 493201052Smarius FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_EN) & 494201052Smarius ~(FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_S | 495201052Smarius FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_S | 496201052Smarius FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_S | 497201052Smarius FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P | 498201052Smarius FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P | 499201052Smarius FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P)); 500201052Smarius 501201052Smarius /* MMU initialization */ 502201052Smarius FIRE_PCI_SET(sc, FO_PCI_MMU_ERR_STAT_CLR, 503201052Smarius FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK); 504201052Smarius /* not enabled by OpenSolaris */ 505201052Smarius FIRE_PCI_SET(sc, FO_PCI_MMU_INT_EN, 506201052Smarius FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK); 507201052Smarius 508201052Smarius /* DMC initialization */ 509201052Smarius FIRE_PCI_SET(sc, FO_PCI_DMC_CORE_BLOCK_INT_EN, ~0ULL); 510201052Smarius FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTA, 0); 511201052Smarius FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTB, 0); 512201052Smarius 513201052Smarius /* PEC initialization */ 514201052Smarius FIRE_PCI_SET(sc, FO_PCI_PEC_CORE_BLOCK_INT_EN, ~0ULL); 515201052Smarius 516201052Smarius /* Establish handlers for interesting interrupts. */ 517201052Smarius if ((ino_bitmap & (1ULL << FO_DMC_PEC_INO)) != 0) 518201052Smarius fire_set_intr(sc, 1, FO_DMC_PEC_INO, fire_dmc_pec, sc); 519201052Smarius if ((ino_bitmap & (1ULL << FO_XCB_INO)) != 0) 520201052Smarius fire_set_intr(sc, 0, FO_XCB_INO, fire_xcb, sc); 521201052Smarius 522201052Smarius /* MSI/MSI-X support */ 523201052Smarius if (OF_getprop(node, "#msi", &sc->sc_msi_count, 524201052Smarius sizeof(sc->sc_msi_count)) == -1) 525201052Smarius panic("%s: could not determine MSI count", __func__); 526201052Smarius if (OF_getprop(node, "msi-ranges", &msi_ranges, 527201052Smarius sizeof(msi_ranges)) == -1) 528201052Smarius sc->sc_msi_first = 0; 529201052Smarius else 530201052Smarius sc->sc_msi_first = msi_ranges.first; 531201052Smarius if (OF_getprop(node, "msi-data-mask", &sc->sc_msi_data_mask, 532201052Smarius sizeof(sc->sc_msi_data_mask)) == -1) 533201052Smarius panic("%s: could not determine MSI data mask", __func__); 534201052Smarius if (OF_getprop(node, "msix-data-width", &sc->sc_msix_data_width, 535201052Smarius sizeof(sc->sc_msix_data_width)) > 0) 536201052Smarius sc->sc_flags |= FIRE_MSIX; 537201052Smarius if (OF_getprop(node, "msi-address-ranges", &msi_addr_ranges, 538201052Smarius sizeof(msi_addr_ranges)) == -1) 539201052Smarius panic("%s: could not determine MSI address ranges", __func__); 540201052Smarius sc->sc_msi_addr32 = OFW_PCI_MSI_ADDR_RANGE_32(&msi_addr_ranges); 541201052Smarius sc->sc_msi_addr64 = OFW_PCI_MSI_ADDR_RANGE_64(&msi_addr_ranges); 542201052Smarius if (OF_getprop(node, "#msi-eqs", &sc->sc_msiq_count, 543201052Smarius sizeof(sc->sc_msiq_count)) == -1) 544201052Smarius panic("%s: could not determine MSI event queue count", 545201052Smarius __func__); 546201052Smarius if (OF_getprop(node, "msi-eq-size", &sc->sc_msiq_size, 547201052Smarius sizeof(sc->sc_msiq_size)) == -1) 548201052Smarius panic("%s: could not determine MSI event queue size", 549201052Smarius __func__); 550201052Smarius if (OF_getprop(node, "msi-eq-to-devino", &msi_eq_to_devino, 551201052Smarius sizeof(msi_eq_to_devino)) == -1 && 552201052Smarius OF_getprop(node, "msi-eq-devino", &msi_eq_to_devino, 553201052Smarius sizeof(msi_eq_to_devino)) == -1) { 554201052Smarius sc->sc_msiq_first = 0; 555201052Smarius sc->sc_msiq_ino_first = FO_EQ_FIRST_INO; 556201052Smarius } else { 557201052Smarius sc->sc_msiq_first = msi_eq_to_devino.eq_first; 558201052Smarius sc->sc_msiq_ino_first = msi_eq_to_devino.devino_first; 559201052Smarius } 560201052Smarius if (sc->sc_msiq_ino_first < FO_EQ_FIRST_INO || 561201052Smarius sc->sc_msiq_ino_first + sc->sc_msiq_count - 1 > FO_EQ_LAST_INO) 562201052Smarius panic("%s: event queues exceed INO range", __func__); 563201052Smarius sc->sc_msi_bitmap = malloc(roundup2(sc->sc_msi_count, NBBY) / NBBY, 564201052Smarius M_DEVBUF, M_NOWAIT | M_ZERO); 565201052Smarius if (sc->sc_msi_bitmap == NULL) 566201052Smarius panic("%s: could not malloc MSI bitmap", __func__); 567201052Smarius sc->sc_msi_msiq_table = malloc(sc->sc_msi_count * 568201052Smarius sizeof(*sc->sc_msi_msiq_table), M_DEVBUF, M_NOWAIT | M_ZERO); 569201052Smarius if (sc->sc_msi_msiq_table == NULL) 570201052Smarius panic("%s: could not malloc MSI-MSI event queue table", 571201052Smarius __func__); 572201052Smarius sc->sc_msiq_bitmap = malloc(roundup2(sc->sc_msiq_count, NBBY) / NBBY, 573201052Smarius M_DEVBUF, M_NOWAIT | M_ZERO); 574201052Smarius if (sc->sc_msiq_bitmap == NULL) 575201052Smarius panic("%s: could not malloc MSI event queue bitmap", __func__); 576201052Smarius j = FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * sc->sc_msiq_count; 577201052Smarius sc->sc_msiq = contigmalloc(j, M_DEVBUF, M_NOWAIT, 0, ~0UL, 578201052Smarius FO_EQ_ALIGNMENT, 0); 579201052Smarius if (sc->sc_msiq == NULL) 580201052Smarius panic("%s: could not contigmalloc MSI event queue", __func__); 581201052Smarius memset(sc->sc_msiq, 0, j); 582201052Smarius FIRE_PCI_SET(sc, FO_PCI_EQ_BASE_ADDR, FO_PCI_EQ_BASE_ADDR_BYPASS | 583201052Smarius (pmap_kextract((vm_offset_t)sc->sc_msiq) & 584201052Smarius FO_PCI_EQ_BASE_ADDR_MASK)); 585201052Smarius for (i = 0; i < sc->sc_msi_count; i++) { 586201052Smarius j = (i + sc->sc_msi_first) << 3; 587201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + j, 588201052Smarius FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + j) & 589201052Smarius ~FO_PCI_MSI_MAP_V); 590201052Smarius } 591201052Smarius for (i = 0; i < sc->sc_msiq_count; i++) { 592201052Smarius j = i + sc->sc_msiq_ino_first; 593201052Smarius if ((ino_bitmap & (1ULL << j)) == 0) { 594201052Smarius mtx_lock(&sc->sc_msi_mtx); 595201052Smarius setbit(sc->sc_msiq_bitmap, i); 596201052Smarius mtx_unlock(&sc->sc_msi_mtx); 597201052Smarius } 598201052Smarius fmqa = intr_vectors[INTMAP_VEC(sc->sc_ign, j)].iv_icarg; 599201052Smarius mtx_init(&fmqa->fmqa_mtx, "msiq_mtx", NULL, MTX_SPIN); 600201052Smarius fmqa->fmqa_base = 601201052Smarius (struct fo_msiq_record *)((caddr_t)sc->sc_msiq + 602201052Smarius (FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * i)); 603201052Smarius j = i + sc->sc_msiq_first; 604201052Smarius fmqa->fmqa_msiq = j; 605201052Smarius j <<= 3; 606201052Smarius fmqa->fmqa_head = FO_PCI_EQ_HD_BASE + j; 607201052Smarius fmqa->fmqa_tail = FO_PCI_EQ_TL_BASE + j; 608201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + j, 609201052Smarius FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I | 610201052Smarius FO_PCI_EQ_CTRL_CLR_DIS); 611201052Smarius FIRE_PCI_WRITE_8(sc, fmqa->fmqa_tail, 612201052Smarius (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK); 613201052Smarius FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, 614201052Smarius (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK); 615201052Smarius } 616201052Smarius FIRE_PCI_SET(sc, FO_PCI_MSI_32_BIT_ADDR, sc->sc_msi_addr32 & 617201052Smarius FO_PCI_MSI_32_BIT_ADDR_MASK); 618201052Smarius FIRE_PCI_SET(sc, FO_PCI_MSI_64_BIT_ADDR, sc->sc_msi_addr64 & 619201052Smarius FO_PCI_MSI_64_BIT_ADDR_MASK); 620201052Smarius 621201052Smarius /* 622201052Smarius * Establish a handler for interesting PCIe messages and disable 623201052Smarius * unintersting ones. 624201052Smarius */ 625201052Smarius mtx_lock(&sc->sc_msi_mtx); 626201052Smarius for (i = 0; i < sc->sc_msiq_count; i++) { 627201052Smarius if (isclr(sc->sc_msiq_bitmap, i) != 0) { 628201052Smarius j = i; 629201052Smarius break; 630201052Smarius } 631201052Smarius } 632201052Smarius if (i == sc->sc_msiq_count) { 633201052Smarius mtx_unlock(&sc->sc_msi_mtx); 634201052Smarius panic("%s: no spare event queue for PCIe messages", __func__); 635201052Smarius } 636201052Smarius setbit(sc->sc_msiq_bitmap, j); 637201052Smarius mtx_unlock(&sc->sc_msi_mtx); 638201052Smarius i = INTMAP_VEC(sc->sc_ign, j + sc->sc_msiq_ino_first); 639201052Smarius if (bus_set_resource(dev, SYS_RES_IRQ, 2, i, 1) != 0) 640201052Smarius panic("%s: failed to add interrupt for PCIe messages", 641201052Smarius __func__); 642201052Smarius fire_set_intr(sc, 2, INTINO(i), fire_pcie, intr_vectors[i].iv_icarg); 643201052Smarius j += sc->sc_msiq_first; 644201052Smarius /* 645201052Smarius * "Please note that setting the EQNUM field to a value larger than 646201052Smarius * 35 will yield unpredictable results." 647201052Smarius */ 648201052Smarius if (j > 35) 649201052Smarius panic("%s: invalid queue for PCIe messages (%d)", 650201052Smarius __func__, j); 651201052Smarius FIRE_PCI_SET(sc, FO_PCI_ERR_COR, FO_PCI_ERR_PME_V | 652201052Smarius ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); 653201052Smarius FIRE_PCI_SET(sc, FO_PCI_ERR_NONFATAL, FO_PCI_ERR_PME_V | 654201052Smarius ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); 655201052Smarius FIRE_PCI_SET(sc, FO_PCI_ERR_FATAL, FO_PCI_ERR_PME_V | 656201052Smarius ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); 657201052Smarius FIRE_PCI_SET(sc, FO_PCI_PM_PME, 0); 658201052Smarius FIRE_PCI_SET(sc, FO_PCI_PME_TO_ACK, 0); 659201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (j << 3), 660201052Smarius FO_PCI_EQ_CTRL_SET_EN); 661201052Smarius 662201052Smarius#define TC_COUNTER_MAX_MASK 0xffffffff 663201052Smarius 664201052Smarius /* 665201052Smarius * Setup JBC/UBC performance counter 0 in bus cycle counting 666223960Smarius * mode as timecounter. 667201052Smarius */ 668201052Smarius if (device_get_unit(dev) == 0) { 669201052Smarius FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT0, 0); 670201052Smarius FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT1, 0); 671201052Smarius FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT_SEL, 672201052Smarius (FO_XBC_PRF_CNT_NONE << FO_XBC_PRF_CNT_CNT1_SHFT) | 673201052Smarius (FO_XBC_PRF_CNT_XB_CLK << FO_XBC_PRF_CNT_CNT0_SHFT)); 674201052Smarius tc = malloc(sizeof(*tc), M_DEVBUF, M_NOWAIT | M_ZERO); 675201052Smarius if (tc == NULL) 676201052Smarius panic("%s: could not malloc timecounter", __func__); 677201052Smarius tc->tc_get_timecount = fire_get_timecount; 678201052Smarius tc->tc_counter_mask = TC_COUNTER_MAX_MASK; 679201052Smarius if (OF_getprop(OF_peer(0), "clock-frequency", &prop, 680201052Smarius sizeof(prop)) == -1) 681201052Smarius panic("%s: could not determine clock frequency", 682201052Smarius __func__); 683201052Smarius tc->tc_frequency = prop; 684201052Smarius tc->tc_name = strdup(device_get_nameunit(dev), M_DEVBUF); 685201052Smarius tc->tc_priv = sc; 686223960Smarius /* 687223960Smarius * Due to initial problems with the JBus-driven performance 688223960Smarius * counters not advancing which might be firmware dependent 689223960Smarius * ensure that it actually works. 690223960Smarius */ 691223960Smarius if (fire_get_timecount(tc) - fire_get_timecount(tc) != 0) 692223960Smarius tc->tc_quality = FIRE_PERF_CNT_QLTY; 693223960Smarius else 694223960Smarius tc->tc_quality = -FIRE_PERF_CNT_QLTY; 695201052Smarius tc_init(tc); 696201052Smarius } 697201052Smarius 698201052Smarius /* 699201052Smarius * Set up the IOMMU. Both Fire and Oberon have one per PBM, but 700201052Smarius * neither has a streaming buffer. 701201052Smarius */ 702201052Smarius memcpy(&sc->sc_dma_methods, &iommu_dma_methods, 703201052Smarius sizeof(sc->sc_dma_methods)); 704201052Smarius sc->sc_is.is_flags = IOMMU_FIRE | IOMMU_PRESERVE_PROM; 705201052Smarius if (sc->sc_mode == FIRE_MODE_OBERON) { 706201052Smarius sc->sc_is.is_flags |= IOMMU_FLUSH_CACHE; 707201052Smarius sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(OBERON_IOMMU_BITS); 708201052Smarius } else { 709201052Smarius sc->sc_dma_methods.dm_dmamap_sync = fire_dmamap_sync; 710201052Smarius sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(FIRE_IOMMU_BITS); 711201052Smarius } 712201052Smarius sc->sc_is.is_sb[0] = sc->sc_is.is_sb[1] = 0; 713201052Smarius /* Punch in our copies. */ 714201052Smarius sc->sc_is.is_bustag = rman_get_bustag(sc->sc_mem_res[FIRE_PCI]); 715201052Smarius sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[FIRE_PCI]); 716201052Smarius sc->sc_is.is_iommu = FO_PCI_MMU; 717201052Smarius val = FIRE_PCI_READ_8(sc, FO_PCI_MMU + IMR_CTL); 718201052Smarius iommu_init(device_get_nameunit(sc->sc_dev), &sc->sc_is, 7, -1, 0); 719201052Smarius#ifdef FIRE_DEBUG 720201052Smarius device_printf(dev, "FO_PCI_MMU + IMR_CTL 0x%016llx -> 0x%016llx\n", 721201052Smarius (long long unsigned)val, (long long unsigned)sc->sc_is.is_cr); 722201052Smarius#endif 723201052Smarius 724201052Smarius /* Initialize memory and I/O rmans. */ 725201052Smarius sc->sc_pci_io_rman.rm_type = RMAN_ARRAY; 726201052Smarius sc->sc_pci_io_rman.rm_descr = "Fire PCI I/O Ports"; 727201052Smarius if (rman_init(&sc->sc_pci_io_rman) != 0 || 728201052Smarius rman_manage_region(&sc->sc_pci_io_rman, 0, FO_IO_SIZE) != 0) 729201052Smarius panic("%s: failed to set up I/O rman", __func__); 730201052Smarius sc->sc_pci_mem_rman.rm_type = RMAN_ARRAY; 731201052Smarius sc->sc_pci_mem_rman.rm_descr = "Fire PCI Memory"; 732201052Smarius if (rman_init(&sc->sc_pci_mem_rman) != 0 || 733201052Smarius rman_manage_region(&sc->sc_pci_mem_rman, 0, FO_MEM_SIZE) != 0) 734201052Smarius panic("%s: failed to set up memory rman", __func__); 735201052Smarius 736201200Smarius i = OF_getprop_alloc(node, "ranges", sizeof(*range), (void **)&range); 737201052Smarius /* 738201052Smarius * Make sure that the expected ranges are present. The 739201052Smarius * OFW_PCI_CS_MEM64 one is not currently used though. 740201052Smarius */ 741201200Smarius if (i != FIRE_NRANGE) 742201052Smarius panic("%s: unsupported number of ranges", __func__); 743201052Smarius /* 744201052Smarius * Find the addresses of the various bus spaces. 745201052Smarius * There should not be multiple ones of one kind. 746201052Smarius * The physical start addresses of the ranges are the configuration, 747201052Smarius * memory and I/O handles. 748201052Smarius */ 749201052Smarius for (i = 0; i < FIRE_NRANGE; i++) { 750201052Smarius j = OFW_PCI_RANGE_CS(&range[i]); 751201052Smarius if (sc->sc_pci_bh[j] != 0) 752201052Smarius panic("%s: duplicate range for space %d", 753201200Smarius __func__, j); 754201052Smarius sc->sc_pci_bh[j] = OFW_PCI_RANGE_PHYS(&range[i]); 755201052Smarius } 756201052Smarius free(range, M_OFWPROP); 757201052Smarius 758201052Smarius /* Allocate our tags. */ 759225931Smarius sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, rman_get_bustag( 760225931Smarius sc->sc_mem_res[FIRE_PCI]), PCI_IO_BUS_SPACE, NULL); 761225931Smarius if (sc->sc_pci_iot == NULL) 762225931Smarius panic("%s: could not allocate PCI I/O tag", __func__); 763225931Smarius sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, rman_get_bustag( 764225931Smarius sc->sc_mem_res[FIRE_PCI]), PCI_CONFIG_BUS_SPACE, NULL); 765225931Smarius if (sc->sc_pci_cfgt == NULL) 766225931Smarius panic("%s: could not allocate PCI configuration space tag", 767225931Smarius __func__); 768233421Smarius if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0x100000000, 769201052Smarius sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr, 770201052Smarius 0xff, 0xffffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0) 771225931Smarius panic("%s: could not create PCI DMA tag", __func__); 772201052Smarius /* Customize the tag. */ 773201052Smarius sc->sc_pci_dmat->dt_cookie = &sc->sc_is; 774201052Smarius sc->sc_pci_dmat->dt_mt = &sc->sc_dma_methods; 775201052Smarius 776201052Smarius /* 777201052Smarius * Get the bus range from the firmware. 778201052Smarius * NB: Neither Fire nor Oberon support PCI bus reenumeration. 779201052Smarius */ 780201052Smarius i = OF_getprop(node, "bus-range", (void *)prop_array, 781201052Smarius sizeof(prop_array)); 782201052Smarius if (i == -1) 783201052Smarius panic("%s: could not get bus-range", __func__); 784201052Smarius if (i != sizeof(prop_array)) 785201052Smarius panic("%s: broken bus-range (%d)", __func__, i); 786201052Smarius sc->sc_pci_secbus = prop_array[0]; 787201052Smarius sc->sc_pci_subbus = prop_array[1]; 788201052Smarius if (bootverbose != 0) 789201052Smarius device_printf(dev, "bus range %u to %u; PCI bus %d\n", 790201052Smarius sc->sc_pci_secbus, sc->sc_pci_subbus, sc->sc_pci_secbus); 791201052Smarius 792201052Smarius ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(ofw_pci_intr_t)); 793201052Smarius 794201052Smarius#define FIRE_SYSCTL_ADD_UINT(name, arg, desc) \ 795201052Smarius SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), \ 796201052Smarius SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, \ 797201052Smarius (name), CTLFLAG_RD, (arg), 0, (desc)) 798201052Smarius 799201052Smarius FIRE_SYSCTL_ADD_UINT("ilu_err", &sc->sc_stats_ilu_err, 800201052Smarius "ILU unknown errors"); 801201052Smarius FIRE_SYSCTL_ADD_UINT("jbc_ce_async", &sc->sc_stats_jbc_ce_async, 802201052Smarius "JBC correctable errors"); 803201052Smarius FIRE_SYSCTL_ADD_UINT("jbc_unsol_int", &sc->sc_stats_jbc_unsol_int, 804201052Smarius "JBC unsolicited interrupt ACK/NACK errors"); 805201052Smarius FIRE_SYSCTL_ADD_UINT("jbc_unsol_rd", &sc->sc_stats_jbc_unsol_rd, 806201052Smarius "JBC unsolicited read response errors"); 807201052Smarius FIRE_SYSCTL_ADD_UINT("mmu_err", &sc->sc_stats_mmu_err, "MMU errors"); 808201052Smarius FIRE_SYSCTL_ADD_UINT("tlu_ce", &sc->sc_stats_tlu_ce, 809201052Smarius "DLU/TLU correctable errors"); 810201052Smarius FIRE_SYSCTL_ADD_UINT("tlu_oe_non_fatal", 811201052Smarius &sc->sc_stats_tlu_oe_non_fatal, 812201052Smarius "DLU/TLU other event non-fatal errors summary"), 813201052Smarius FIRE_SYSCTL_ADD_UINT("tlu_oe_rx_err", &sc->sc_stats_tlu_oe_rx_err, 814201052Smarius "DLU/TLU receive other event errors"), 815201052Smarius FIRE_SYSCTL_ADD_UINT("tlu_oe_tx_err", &sc->sc_stats_tlu_oe_tx_err, 816201052Smarius "DLU/TLU transmit other event errors"), 817201052Smarius FIRE_SYSCTL_ADD_UINT("ubc_dmardue", &sc->sc_stats_ubc_dmardue, 818201052Smarius "UBC DMARDUE erros"); 819201052Smarius 820201052Smarius#undef FIRE_SYSCTL_ADD_UINT 821201052Smarius 822201052Smarius device_add_child(dev, "pci", -1); 823201052Smarius return (bus_generic_attach(dev)); 824201052Smarius} 825201052Smarius 826201052Smariusstatic void 827201052Smariusfire_set_intr(struct fire_softc *sc, u_int index, u_int ino, 828201052Smarius driver_filter_t handler, void *arg) 829201052Smarius{ 830201052Smarius u_long vec; 831201052Smarius int rid; 832201052Smarius 833201052Smarius rid = index; 834201052Smarius sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev, 835201052Smarius SYS_RES_IRQ, &rid, RF_ACTIVE); 836201052Smarius if (sc->sc_irq_res[index] == NULL || 837201052Smarius INTINO(vec = rman_get_start(sc->sc_irq_res[index])) != ino || 838201052Smarius INTIGN(vec) != sc->sc_ign || 839201052Smarius intr_vectors[vec].iv_ic != &fire_ic || 840201052Smarius bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index], 841216961Smarius INTR_TYPE_MISC | INTR_BRIDGE, handler, NULL, arg, 842201052Smarius &sc->sc_ihand[index]) != 0) 843201052Smarius panic("%s: failed to set up interrupt %d", __func__, index); 844201052Smarius} 845201052Smarius 846201052Smariusstatic int 847201052Smariusfire_intr_register(struct fire_softc *sc, u_int ino) 848201052Smarius{ 849201052Smarius struct fire_icarg *fica; 850201052Smarius bus_addr_t intrclr, intrmap; 851201052Smarius int error; 852201052Smarius 853201052Smarius if (fire_get_intrmap(sc, ino, &intrmap, &intrclr) == 0) 854201052Smarius return (ENXIO); 855201052Smarius fica = malloc((ino >= FO_EQ_FIRST_INO && ino <= FO_EQ_LAST_INO) ? 856201052Smarius sizeof(struct fire_msiqarg) : sizeof(struct fire_icarg), M_DEVBUF, 857203094Smarius M_NOWAIT | M_ZERO); 858201052Smarius if (fica == NULL) 859201052Smarius return (ENOMEM); 860201052Smarius fica->fica_sc = sc; 861201052Smarius fica->fica_map = intrmap; 862201052Smarius fica->fica_clr = intrclr; 863201052Smarius error = (intr_controller_register(INTMAP_VEC(sc->sc_ign, ino), 864201052Smarius &fire_ic, fica)); 865201052Smarius if (error != 0) 866201052Smarius free(fica, M_DEVBUF); 867201052Smarius return (error); 868201052Smarius} 869201052Smarius 870201052Smariusstatic int 871201052Smariusfire_get_intrmap(struct fire_softc *sc, u_int ino, bus_addr_t *intrmapptr, 872201052Smarius bus_addr_t *intrclrptr) 873201052Smarius{ 874201052Smarius 875201052Smarius if (ino > FO_MAX_INO) { 876201052Smarius device_printf(sc->sc_dev, "out of range INO %d requested\n", 877201052Smarius ino); 878201052Smarius return (0); 879201052Smarius } 880201052Smarius 881201052Smarius ino <<= 3; 882201052Smarius if (intrmapptr != NULL) 883201052Smarius *intrmapptr = FO_PCI_INT_MAP_BASE + ino; 884201052Smarius if (intrclrptr != NULL) 885201052Smarius *intrclrptr = FO_PCI_INT_CLR_BASE + ino; 886201052Smarius return (1); 887201052Smarius} 888201052Smarius 889201052Smarius/* 890201052Smarius * Interrupt handlers 891201052Smarius */ 892201052Smariusstatic int 893201052Smariusfire_dmc_pec(void *arg) 894201052Smarius{ 895201052Smarius struct fire_softc *sc; 896201052Smarius device_t dev; 897201052Smarius uint64_t cestat, dmcstat, ilustat, imustat, mcstat, mmustat, mmutfar; 898201052Smarius uint64_t mmutfsr, oestat, pecstat, uestat, val; 899201052Smarius u_int fatal, oenfatal; 900201052Smarius 901201052Smarius fatal = 0; 902201052Smarius sc = arg; 903201052Smarius dev = sc->sc_dev; 904201052Smarius mtx_lock_spin(&sc->sc_pcib_mtx); 905201052Smarius mcstat = FIRE_PCI_READ_8(sc, FO_PCI_MULTI_CORE_ERR_STAT); 906201052Smarius if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_DMC) != 0) { 907201052Smarius dmcstat = FIRE_PCI_READ_8(sc, FO_PCI_DMC_CORE_BLOCK_ERR_STAT); 908201052Smarius if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_IMU) != 0) { 909201052Smarius imustat = FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_STAT); 910201052Smarius device_printf(dev, "IMU error %#llx\n", 911201052Smarius (unsigned long long)imustat); 912201052Smarius if ((imustat & 913201052Smarius FO_PCI_IMU_ERR_INT_EQ_NOT_EN_P) != 0) { 914201052Smarius fatal = 1; 915201052Smarius val = FIRE_PCI_READ_8(sc, 916201052Smarius FO_PCI_IMU_SCS_ERR_LOG); 917201052Smarius device_printf(dev, "SCS error log %#llx\n", 918201052Smarius (unsigned long long)val); 919201052Smarius } 920201052Smarius if ((imustat & FO_PCI_IMU_ERR_INT_EQ_OVER_P) != 0) { 921201052Smarius fatal = 1; 922201052Smarius val = FIRE_PCI_READ_8(sc, 923201052Smarius FO_PCI_IMU_EQS_ERR_LOG); 924201052Smarius device_printf(dev, "EQS error log %#llx\n", 925201052Smarius (unsigned long long)val); 926201052Smarius } 927201052Smarius if ((imustat & (FO_PCI_IMU_ERR_INT_MSI_MAL_ERR_P | 928201052Smarius FO_PCI_IMU_ERR_INT_MSI_PAR_ERR_P | 929201052Smarius FO_PCI_IMU_ERR_INT_PMEACK_MES_NOT_EN_P | 930201052Smarius FO_PCI_IMU_ERR_INT_PMPME_MES_NOT_EN_P | 931201052Smarius FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P | 932201052Smarius FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P | 933201052Smarius FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P | 934201052Smarius FO_PCI_IMU_ERR_INT_MSI_NOT_EN_P)) != 0) { 935201052Smarius fatal = 1; 936201052Smarius val = FIRE_PCI_READ_8(sc, 937201052Smarius FO_PCI_IMU_RDS_ERR_LOG); 938201052Smarius device_printf(dev, "RDS error log %#llx\n", 939201052Smarius (unsigned long long)val); 940201052Smarius } 941201052Smarius } 942201052Smarius if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_MMU) != 0) { 943201052Smarius fatal = 1; 944201052Smarius mmustat = FIRE_PCI_READ_8(sc, FO_PCI_MMU_INT_STAT); 945201052Smarius mmutfar = FIRE_PCI_READ_8(sc, 946201052Smarius FO_PCI_MMU_TRANS_FAULT_ADDR); 947201052Smarius mmutfsr = FIRE_PCI_READ_8(sc, 948201052Smarius FO_PCI_MMU_TRANS_FAULT_STAT); 949201052Smarius if ((mmustat & (FO_PCI_MMU_ERR_INT_TBW_DPE_P | 950201052Smarius FO_PCI_MMU_ERR_INT_TBW_ERR_P | 951201052Smarius FO_PCI_MMU_ERR_INT_TBW_UDE_P | 952201052Smarius FO_PCI_MMU_ERR_INT_TBW_DME_P | 953201052Smarius FO_PCI_MMU_ERR_INT_TTC_CAE_P | 954201052Smarius FIRE_PCI_MMU_ERR_INT_TTC_DPE_P | 955201052Smarius OBERON_PCI_MMU_ERR_INT_TTC_DUE_P | 956201052Smarius FO_PCI_MMU_ERR_INT_TRN_ERR_P)) != 0) 957201052Smarius fatal = 1; 958201052Smarius else { 959201052Smarius sc->sc_stats_mmu_err++; 960201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_MMU_ERR_STAT_CLR, 961201052Smarius mmustat); 962201052Smarius } 963201052Smarius device_printf(dev, 964201052Smarius "MMU error %#llx: TFAR %#llx TFSR %#llx\n", 965201052Smarius (unsigned long long)mmustat, 966201052Smarius (unsigned long long)mmutfar, 967201052Smarius (unsigned long long)mmutfsr); 968201052Smarius } 969201052Smarius } 970201052Smarius if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_PEC) != 0) { 971201052Smarius pecstat = FIRE_PCI_READ_8(sc, FO_PCI_PEC_CORE_BLOCK_INT_STAT); 972201052Smarius if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_UERR) != 0) { 973201052Smarius fatal = 1; 974201052Smarius uestat = FIRE_PCI_READ_8(sc, 975201052Smarius FO_PCI_TLU_UERR_INT_STAT); 976201052Smarius device_printf(dev, 977201052Smarius "DLU/TLU uncorrectable error %#llx\n", 978201052Smarius (unsigned long long)uestat); 979201052Smarius if ((uestat & (FO_PCI_TLU_UERR_INT_UR_P | 980201052Smarius OBERON_PCI_TLU_UERR_INT_POIS_P | 981201052Smarius FO_PCI_TLU_UERR_INT_MFP_P | 982201052Smarius FO_PCI_TLU_UERR_INT_ROF_P | 983201052Smarius FO_PCI_TLU_UERR_INT_UC_P | 984201052Smarius FIRE_PCI_TLU_UERR_INT_PP_P | 985201052Smarius OBERON_PCI_TLU_UERR_INT_POIS_P)) != 0) { 986201052Smarius val = FIRE_PCI_READ_8(sc, 987201052Smarius FO_PCI_TLU_RX_UERR_HDR1_LOG); 988201052Smarius device_printf(dev, 989201052Smarius "receive header log %#llx\n", 990201052Smarius (unsigned long long)val); 991201052Smarius val = FIRE_PCI_READ_8(sc, 992201052Smarius FO_PCI_TLU_RX_UERR_HDR2_LOG); 993201052Smarius device_printf(dev, 994201052Smarius "receive header log 2 %#llx\n", 995201052Smarius (unsigned long long)val); 996201052Smarius } 997201052Smarius if ((uestat & FO_PCI_TLU_UERR_INT_CTO_P) != 0) { 998201052Smarius val = FIRE_PCI_READ_8(sc, 999201052Smarius FO_PCI_TLU_TX_UERR_HDR1_LOG); 1000201052Smarius device_printf(dev, 1001201052Smarius "transmit header log %#llx\n", 1002201052Smarius (unsigned long long)val); 1003201052Smarius val = FIRE_PCI_READ_8(sc, 1004201052Smarius FO_PCI_TLU_TX_UERR_HDR2_LOG); 1005201052Smarius device_printf(dev, 1006201052Smarius "transmit header log 2 %#llx\n", 1007201052Smarius (unsigned long long)val); 1008201052Smarius } 1009201052Smarius if ((uestat & FO_PCI_TLU_UERR_INT_DLP_P) != 0) { 1010201052Smarius val = FIRE_PCI_READ_8(sc, 1011201052Smarius FO_PCI_LPU_LNK_LYR_INT_STAT); 1012201052Smarius device_printf(dev, 1013201052Smarius "link layer interrupt and status %#llx\n", 1014201052Smarius (unsigned long long)val); 1015201052Smarius } 1016201052Smarius if ((uestat & FO_PCI_TLU_UERR_INT_TE_P) != 0) { 1017201052Smarius val = FIRE_PCI_READ_8(sc, 1018201052Smarius FO_PCI_LPU_PHY_LYR_INT_STAT); 1019201052Smarius device_printf(dev, 1020201052Smarius "phy layer interrupt and status %#llx\n", 1021201052Smarius (unsigned long long)val); 1022201052Smarius } 1023201052Smarius } 1024201052Smarius if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_CERR) != 0) { 1025201052Smarius sc->sc_stats_tlu_ce++; 1026201052Smarius cestat = FIRE_PCI_READ_8(sc, 1027201052Smarius FO_PCI_TLU_CERR_INT_STAT); 1028201052Smarius device_printf(dev, 1029201052Smarius "DLU/TLU correctable error %#llx\n", 1030201052Smarius (unsigned long long)cestat); 1031201052Smarius val = FIRE_PCI_READ_8(sc, 1032201052Smarius FO_PCI_LPU_LNK_LYR_INT_STAT); 1033201052Smarius device_printf(dev, 1034201052Smarius "link layer interrupt and status %#llx\n", 1035201052Smarius (unsigned long long)val); 1036201052Smarius if ((cestat & FO_PCI_TLU_CERR_INT_RE_P) != 0) { 1037201052Smarius FIRE_PCI_WRITE_8(sc, 1038201052Smarius FO_PCI_LPU_LNK_LYR_INT_STAT, val); 1039201052Smarius val = FIRE_PCI_READ_8(sc, 1040201052Smarius FO_PCI_LPU_PHY_LYR_INT_STAT); 1041201052Smarius device_printf(dev, 1042201052Smarius "phy layer interrupt and status %#llx\n", 1043201052Smarius (unsigned long long)val); 1044201052Smarius } 1045201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_TLU_CERR_STAT_CLR, 1046201052Smarius cestat); 1047201052Smarius } 1048201052Smarius if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_OEVENT) != 0) { 1049201052Smarius oenfatal = 0; 1050201052Smarius oestat = FIRE_PCI_READ_8(sc, 1051201052Smarius FO_PCI_TLU_OEVENT_INT_STAT); 1052201052Smarius device_printf(dev, "DLU/TLU other event %#llx\n", 1053201052Smarius (unsigned long long)oestat); 1054201052Smarius if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 1055201052Smarius FO_PCI_TLU_OEVENT_MRC_P | 1056201052Smarius FO_PCI_TLU_OEVENT_WUC_P | 1057201052Smarius FO_PCI_TLU_OEVENT_RUC_P | 1058201052Smarius FO_PCI_TLU_OEVENT_CRS_P)) != 0) { 1059201052Smarius val = FIRE_PCI_READ_8(sc, 1060201052Smarius FO_PCI_TLU_RX_OEVENT_HDR1_LOG); 1061201052Smarius device_printf(dev, 1062201052Smarius "receive header log %#llx\n", 1063201052Smarius (unsigned long long)val); 1064201052Smarius val = FIRE_PCI_READ_8(sc, 1065201052Smarius FO_PCI_TLU_RX_OEVENT_HDR2_LOG); 1066201052Smarius device_printf(dev, 1067201052Smarius "receive header log 2 %#llx\n", 1068201052Smarius (unsigned long long)val); 1069201052Smarius if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 1070201052Smarius FO_PCI_TLU_OEVENT_MRC_P | 1071201052Smarius FO_PCI_TLU_OEVENT_WUC_P | 1072201052Smarius FO_PCI_TLU_OEVENT_RUC_P)) != 0) 1073201052Smarius fatal = 1; 1074201052Smarius else { 1075201052Smarius sc->sc_stats_tlu_oe_rx_err++; 1076201052Smarius oenfatal = 1; 1077201052Smarius } 1078201052Smarius } 1079201052Smarius if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 1080201052Smarius FO_PCI_TLU_OEVENT_CTO_P | 1081201052Smarius FO_PCI_TLU_OEVENT_WUC_P | 1082201052Smarius FO_PCI_TLU_OEVENT_RUC_P)) != 0) { 1083201052Smarius val = FIRE_PCI_READ_8(sc, 1084201052Smarius FO_PCI_TLU_TX_OEVENT_HDR1_LOG); 1085201052Smarius device_printf(dev, 1086201052Smarius "transmit header log %#llx\n", 1087201052Smarius (unsigned long long)val); 1088201052Smarius val = FIRE_PCI_READ_8(sc, 1089201052Smarius FO_PCI_TLU_TX_OEVENT_HDR2_LOG); 1090201052Smarius device_printf(dev, 1091201052Smarius "transmit header log 2 %#llx\n", 1092201052Smarius (unsigned long long)val); 1093201052Smarius if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 1094201052Smarius FO_PCI_TLU_OEVENT_CTO_P | 1095201052Smarius FO_PCI_TLU_OEVENT_WUC_P | 1096201052Smarius FO_PCI_TLU_OEVENT_RUC_P)) != 0) 1097201052Smarius fatal = 1; 1098201052Smarius else { 1099201052Smarius sc->sc_stats_tlu_oe_tx_err++; 1100201052Smarius oenfatal = 1; 1101201052Smarius } 1102201052Smarius } 1103201052Smarius if ((oestat & (FO_PCI_TLU_OEVENT_ERO_P | 1104201052Smarius FO_PCI_TLU_OEVENT_EMP_P | 1105201052Smarius FO_PCI_TLU_OEVENT_EPE_P | 1106201052Smarius FIRE_PCI_TLU_OEVENT_ERP_P | 1107201052Smarius OBERON_PCI_TLU_OEVENT_ERBU_P | 1108201052Smarius FIRE_PCI_TLU_OEVENT_EIP_P | 1109201052Smarius OBERON_PCI_TLU_OEVENT_EIUE_P)) != 0) { 1110201052Smarius fatal = 1; 1111201052Smarius val = FIRE_PCI_READ_8(sc, 1112201052Smarius FO_PCI_LPU_LNK_LYR_INT_STAT); 1113201052Smarius device_printf(dev, 1114201052Smarius "link layer interrupt and status %#llx\n", 1115201052Smarius (unsigned long long)val); 1116201052Smarius } 1117201052Smarius if ((oestat & (FO_PCI_TLU_OEVENT_IIP_P | 1118201052Smarius FO_PCI_TLU_OEVENT_EDP_P | 1119201052Smarius FIRE_PCI_TLU_OEVENT_EHP_P | 1120201052Smarius OBERON_PCI_TLU_OEVENT_TLUEITMO_S | 1121201052Smarius FO_PCI_TLU_OEVENT_ERU_P)) != 0) 1122201052Smarius fatal = 1; 1123201052Smarius if ((oestat & (FO_PCI_TLU_OEVENT_NFP_P | 1124201052Smarius FO_PCI_TLU_OEVENT_LWC_P | 1125201052Smarius FO_PCI_TLU_OEVENT_LIN_P | 1126201052Smarius FO_PCI_TLU_OEVENT_LRS_P | 1127201052Smarius FO_PCI_TLU_OEVENT_LDN_P | 1128201052Smarius FO_PCI_TLU_OEVENT_LUP_P)) != 0) 1129201052Smarius oenfatal = 1; 1130201052Smarius if (oenfatal != 0) { 1131201052Smarius sc->sc_stats_tlu_oe_non_fatal++; 1132201052Smarius FIRE_PCI_WRITE_8(sc, 1133201052Smarius FO_PCI_TLU_OEVENT_STAT_CLR, oestat); 1134201052Smarius if ((oestat & FO_PCI_TLU_OEVENT_LIN_P) != 0) 1135201052Smarius FIRE_PCI_WRITE_8(sc, 1136201052Smarius FO_PCI_LPU_LNK_LYR_INT_STAT, 1137201052Smarius FIRE_PCI_READ_8(sc, 1138201052Smarius FO_PCI_LPU_LNK_LYR_INT_STAT)); 1139201052Smarius } 1140201052Smarius } 1141201052Smarius if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_ILU) != 0) { 1142201052Smarius ilustat = FIRE_PCI_READ_8(sc, FO_PCI_ILU_INT_STAT); 1143201052Smarius device_printf(dev, "ILU error %#llx\n", 1144201052Smarius (unsigned long long)ilustat); 1145201052Smarius if ((ilustat & (FIRE_PCI_ILU_ERR_INT_IHB_PE_P | 1146201052Smarius FIRE_PCI_ILU_ERR_INT_IHB_PE_P)) != 0) 1147201052Smarius fatal = 1; 1148201052Smarius else { 1149201052Smarius sc->sc_stats_ilu_err++; 1150201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_ILU_INT_STAT, 1151201052Smarius ilustat); 1152201052Smarius } 1153201052Smarius } 1154201052Smarius } 1155201052Smarius mtx_unlock_spin(&sc->sc_pcib_mtx); 1156201052Smarius if (fatal != 0) 1157201052Smarius panic("%s: fatal DMC/PEC error", 1158201052Smarius device_get_nameunit(sc->sc_dev)); 1159201052Smarius return (FILTER_HANDLED); 1160201052Smarius} 1161201052Smarius 1162201052Smariusstatic int 1163201052Smariusfire_xcb(void *arg) 1164201052Smarius{ 1165201052Smarius struct fire_softc *sc; 1166201052Smarius device_t dev; 1167201052Smarius uint64_t errstat, intstat, val; 1168201052Smarius u_int fatal; 1169201052Smarius 1170201052Smarius fatal = 0; 1171201052Smarius sc = arg; 1172201052Smarius dev = sc->sc_dev; 1173201052Smarius mtx_lock_spin(&sc->sc_pcib_mtx); 1174201052Smarius if (sc->sc_mode == FIRE_MODE_OBERON) { 1175201052Smarius intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT); 1176201052Smarius device_printf(dev, "UBC error: interrupt status %#llx\n", 1177201052Smarius (unsigned long long)intstat); 1178201052Smarius if ((intstat & ~(OBERON_UBC_ERR_INT_DMARDUEB_P | 1179201052Smarius OBERON_UBC_ERR_INT_DMARDUEA_P)) != 0) 1180201052Smarius fatal = 1; 1181201052Smarius else 1182201052Smarius sc->sc_stats_ubc_dmardue++; 1183201052Smarius if (fatal != 0) { 1184201052Smarius mtx_unlock_spin(&sc->sc_pcib_mtx); 1185201052Smarius panic("%s: fatal UBC core block error", 1186201052Smarius device_get_nameunit(sc->sc_dev)); 1187201052Smarius } else { 1188201052Smarius FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); 1189201052Smarius mtx_unlock_spin(&sc->sc_pcib_mtx); 1190201052Smarius } 1191201052Smarius } else { 1192201052Smarius errstat = FIRE_CTRL_READ_8(sc, FIRE_JBC_CORE_BLOCK_ERR_STAT); 1193201052Smarius if ((errstat & (FIRE_JBC_CORE_BLOCK_ERR_STAT_MERGE | 1194201052Smarius FIRE_JBC_CORE_BLOCK_ERR_STAT_JBCINT | 1195201052Smarius FIRE_JBC_CORE_BLOCK_ERR_STAT_DMCINT)) != 0) { 1196201052Smarius intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT); 1197201052Smarius device_printf(dev, "JBC interrupt status %#llx\n", 1198201052Smarius (unsigned long long)intstat); 1199201052Smarius if ((intstat & FIRE_JBC_ERR_INT_EBUS_TO_P) != 0) { 1200201052Smarius val = FIRE_CTRL_READ_8(sc, 1201201052Smarius FIRE_JBC_CSR_ERR_LOG); 1202201052Smarius device_printf(dev, "CSR error log %#llx\n", 1203201052Smarius (unsigned long long)val); 1204201052Smarius } 1205201052Smarius if ((intstat & (FIRE_JBC_ERR_INT_UNSOL_RD_P | 1206201052Smarius FIRE_JBC_ERR_INT_UNSOL_INT_P)) != 0) { 1207201052Smarius if ((intstat & 1208201052Smarius FIRE_JBC_ERR_INT_UNSOL_RD_P) != 0) 1209201052Smarius sc->sc_stats_jbc_unsol_rd++; 1210201052Smarius if ((intstat & 1211201052Smarius FIRE_JBC_ERR_INT_UNSOL_INT_P) != 0) 1212201052Smarius sc->sc_stats_jbc_unsol_int++; 1213201052Smarius val = FIRE_CTRL_READ_8(sc, 1214201052Smarius FIRE_DMCINT_IDC_ERR_LOG); 1215201052Smarius device_printf(dev, 1216201052Smarius "DMCINT IDC error log %#llx\n", 1217201052Smarius (unsigned long long)val); 1218201052Smarius } 1219201052Smarius if ((intstat & (FIRE_JBC_ERR_INT_MB_PER_P | 1220201052Smarius FIRE_JBC_ERR_INT_MB_PEW_P)) != 0) { 1221201052Smarius fatal = 1; 1222201052Smarius val = FIRE_CTRL_READ_8(sc, 1223201052Smarius FIRE_MERGE_TRANS_ERR_LOG); 1224201052Smarius device_printf(dev, 1225201052Smarius "merge transaction error log %#llx\n", 1226201052Smarius (unsigned long long)val); 1227201052Smarius } 1228201052Smarius if ((intstat & FIRE_JBC_ERR_INT_IJP_P) != 0) { 1229201052Smarius fatal = 1; 1230201052Smarius val = FIRE_CTRL_READ_8(sc, 1231201052Smarius FIRE_JBCINT_OTRANS_ERR_LOG); 1232201052Smarius device_printf(dev, 1233201052Smarius "JBCINT out transaction error log " 1234201052Smarius "%#llx\n", (unsigned long long)val); 1235201052Smarius val = FIRE_CTRL_READ_8(sc, 1236201052Smarius FIRE_JBCINT_OTRANS_ERR_LOG2); 1237201052Smarius device_printf(dev, 1238201052Smarius "JBCINT out transaction error log 2 " 1239201052Smarius "%#llx\n", (unsigned long long)val); 1240201052Smarius } 1241201052Smarius if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P | 1242201052Smarius FIRE_JBC_ERR_INT_CE_ASYN_P | 1243201052Smarius FIRE_JBC_ERR_INT_JTE_P | FIRE_JBC_ERR_INT_JBE_P | 1244201052Smarius FIRE_JBC_ERR_INT_JUE_P | 1245201052Smarius FIRE_JBC_ERR_INT_ICISE_P | 1246201052Smarius FIRE_JBC_ERR_INT_WR_DPE_P | 1247201052Smarius FIRE_JBC_ERR_INT_RD_DPE_P | 1248201052Smarius FIRE_JBC_ERR_INT_ILL_BMW_P | 1249201052Smarius FIRE_JBC_ERR_INT_ILL_BMR_P | 1250201052Smarius FIRE_JBC_ERR_INT_BJC_P)) != 0) { 1251201052Smarius if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P | 1252201052Smarius FIRE_JBC_ERR_INT_JTE_P | 1253201052Smarius FIRE_JBC_ERR_INT_JBE_P | 1254201052Smarius FIRE_JBC_ERR_INT_JUE_P | 1255201052Smarius FIRE_JBC_ERR_INT_ICISE_P | 1256201052Smarius FIRE_JBC_ERR_INT_WR_DPE_P | 1257201052Smarius FIRE_JBC_ERR_INT_RD_DPE_P | 1258201052Smarius FIRE_JBC_ERR_INT_ILL_BMW_P | 1259201052Smarius FIRE_JBC_ERR_INT_ILL_BMR_P | 1260201052Smarius FIRE_JBC_ERR_INT_BJC_P)) != 0) 1261201052Smarius fatal = 1; 1262201052Smarius else 1263201052Smarius sc->sc_stats_jbc_ce_async++; 1264201052Smarius val = FIRE_CTRL_READ_8(sc, 1265201052Smarius FIRE_JBCINT_ITRANS_ERR_LOG); 1266201052Smarius device_printf(dev, 1267201052Smarius "JBCINT in transaction error log %#llx\n", 1268201052Smarius (unsigned long long)val); 1269201052Smarius val = FIRE_CTRL_READ_8(sc, 1270201052Smarius FIRE_JBCINT_ITRANS_ERR_LOG2); 1271201052Smarius device_printf(dev, 1272201052Smarius "JBCINT in transaction error log 2 " 1273201052Smarius "%#llx\n", (unsigned long long)val); 1274201052Smarius } 1275201052Smarius if ((intstat & (FIRE_JBC_ERR_INT_PIO_UNMAP_RD_P | 1276201052Smarius FIRE_JBC_ERR_INT_ILL_ACC_RD_P | 1277201052Smarius FIRE_JBC_ERR_INT_PIO_UNMAP_P | 1278201052Smarius FIRE_JBC_ERR_INT_PIO_DPE_P | 1279201052Smarius FIRE_JBC_ERR_INT_PIO_CPE_P | 1280201052Smarius FIRE_JBC_ERR_INT_ILL_ACC_P)) != 0) { 1281201052Smarius fatal = 1; 1282201052Smarius val = FIRE_CTRL_READ_8(sc, 1283201052Smarius FIRE_JBC_CSR_ERR_LOG); 1284201052Smarius device_printf(dev, 1285201052Smarius "DMCINT ODCD error log %#llx\n", 1286201052Smarius (unsigned long long)val); 1287201052Smarius } 1288201052Smarius if ((intstat & (FIRE_JBC_ERR_INT_MB_PEA_P | 1289201052Smarius FIRE_JBC_ERR_INT_CPE_P | FIRE_JBC_ERR_INT_APE_P | 1290201052Smarius FIRE_JBC_ERR_INT_PIO_CPE_P | 1291201052Smarius FIRE_JBC_ERR_INT_JTCEEW_P | 1292201052Smarius FIRE_JBC_ERR_INT_JTCEEI_P | 1293201052Smarius FIRE_JBC_ERR_INT_JTCEER_P)) != 0) { 1294201052Smarius fatal = 1; 1295201052Smarius val = FIRE_CTRL_READ_8(sc, 1296201052Smarius FIRE_FATAL_ERR_LOG); 1297201052Smarius device_printf(dev, "fatal error log %#llx\n", 1298201052Smarius (unsigned long long)val); 1299201052Smarius val = FIRE_CTRL_READ_8(sc, 1300201052Smarius FIRE_FATAL_ERR_LOG2); 1301201052Smarius device_printf(dev, "fatal error log 2 " 1302201052Smarius "%#llx\n", (unsigned long long)val); 1303201052Smarius } 1304201052Smarius if (fatal != 0) { 1305201052Smarius mtx_unlock_spin(&sc->sc_pcib_mtx); 1306201052Smarius panic("%s: fatal JBC core block error", 1307201052Smarius device_get_nameunit(sc->sc_dev)); 1308201052Smarius } else { 1309201052Smarius FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); 1310201052Smarius mtx_unlock_spin(&sc->sc_pcib_mtx); 1311201052Smarius } 1312201052Smarius } else { 1313201052Smarius mtx_unlock_spin(&sc->sc_pcib_mtx); 1314201052Smarius panic("%s: unknown JCB core block error status %#llx", 1315201052Smarius device_get_nameunit(sc->sc_dev), 1316201052Smarius (unsigned long long)errstat); 1317201052Smarius } 1318201052Smarius } 1319201052Smarius return (FILTER_HANDLED); 1320201052Smarius} 1321201052Smarius 1322201052Smariusstatic int 1323201052Smariusfire_pcie(void *arg) 1324201052Smarius{ 1325201052Smarius struct fire_msiqarg *fmqa; 1326201052Smarius struct fire_softc *sc; 1327201052Smarius struct fo_msiq_record *qrec; 1328201052Smarius device_t dev; 1329201052Smarius uint64_t word0; 1330201052Smarius u_int head, msg, msiq; 1331201052Smarius 1332201052Smarius fmqa = arg; 1333201052Smarius sc = fmqa->fmqa_fica.fica_sc; 1334201052Smarius dev = sc->sc_dev; 1335201052Smarius msiq = fmqa->fmqa_msiq; 1336201052Smarius mtx_lock_spin(&fmqa->fmqa_mtx); 1337201052Smarius head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >> 1338201052Smarius FO_PCI_EQ_HD_SHFT; 1339201052Smarius qrec = &fmqa->fmqa_base[head]; 1340201052Smarius word0 = qrec->fomqr_word0; 1341201052Smarius for (;;) { 1342201052Smarius KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSG) != 0, 1343201052Smarius ("%s: received non-PCIe message in event queue %d " 1344201052Smarius "(word0 %#llx)", device_get_nameunit(dev), msiq, 1345201052Smarius (unsigned long long)word0)); 1346201052Smarius msg = (word0 & FO_MQR_WORD0_DATA0_MASK) >> 1347201052Smarius FO_MQR_WORD0_DATA0_SHFT; 1348201052Smarius 1349201052Smarius#define PCIE_MSG_CODE_ERR_COR 0x30 1350201052Smarius#define PCIE_MSG_CODE_ERR_NONFATAL 0x31 1351201052Smarius#define PCIE_MSG_CODE_ERR_FATAL 0x33 1352201052Smarius 1353201052Smarius if (msg == PCIE_MSG_CODE_ERR_COR) 1354201052Smarius device_printf(dev, "correctable PCIe error\n"); 1355201052Smarius else if (msg == PCIE_MSG_CODE_ERR_NONFATAL || 1356201052Smarius msg == PCIE_MSG_CODE_ERR_FATAL) 1357201052Smarius panic("%s: %sfatal PCIe error", 1358201052Smarius device_get_nameunit(dev), 1359201052Smarius msg == PCIE_MSG_CODE_ERR_NONFATAL ? "non-" : ""); 1360201052Smarius else 1361201052Smarius panic("%s: received unknown PCIe message %#x", 1362201052Smarius device_get_nameunit(dev), msg); 1363201052Smarius qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK; 1364201052Smarius head = (head + 1) % sc->sc_msiq_size; 1365201052Smarius qrec = &fmqa->fmqa_base[head]; 1366201052Smarius word0 = qrec->fomqr_word0; 1367201052Smarius if (__predict_true((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0)) 1368201052Smarius break; 1369201052Smarius } 1370201052Smarius FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) << 1371201052Smarius FO_PCI_EQ_HD_SHFT); 1372201052Smarius if ((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) & 1373201052Smarius FO_PCI_EQ_TL_OVERR) != 0) { 1374201052Smarius device_printf(dev, "event queue %d overflow\n", msiq); 1375201052Smarius msiq <<= 3; 1376201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, 1377201052Smarius FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) | 1378201052Smarius FO_PCI_EQ_CTRL_CLR_COVERR); 1379201052Smarius } 1380201052Smarius mtx_unlock_spin(&fmqa->fmqa_mtx); 1381201052Smarius return (FILTER_HANDLED); 1382201052Smarius} 1383201052Smarius 1384201052Smariusstatic int 1385201052Smariusfire_maxslots(device_t dev) 1386201052Smarius{ 1387201052Smarius 1388201052Smarius return (1); 1389201052Smarius} 1390201052Smarius 1391201052Smariusstatic uint32_t 1392201052Smariusfire_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, 1393201052Smarius int width) 1394201052Smarius{ 1395201052Smarius struct fire_softc *sc; 1396201052Smarius bus_space_handle_t bh; 1397201052Smarius u_long offset = 0; 1398201052Smarius uint32_t r, wrd; 1399201052Smarius int i; 1400201052Smarius uint16_t shrt; 1401201052Smarius uint8_t byte; 1402201052Smarius 1403201052Smarius sc = device_get_softc(dev); 1404201052Smarius if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus || 1405201052Smarius slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCIE_REGMAX) 1406201052Smarius return (-1); 1407201052Smarius 1408201052Smarius offset = FO_CONF_OFF(bus, slot, func, reg); 1409201052Smarius bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG]; 1410201052Smarius switch (width) { 1411201052Smarius case 1: 1412201052Smarius i = bus_space_peek_1(sc->sc_pci_cfgt, bh, offset, &byte); 1413201052Smarius r = byte; 1414201052Smarius break; 1415201052Smarius case 2: 1416201052Smarius i = bus_space_peek_2(sc->sc_pci_cfgt, bh, offset, &shrt); 1417201052Smarius r = shrt; 1418201052Smarius break; 1419201052Smarius case 4: 1420201052Smarius i = bus_space_peek_4(sc->sc_pci_cfgt, bh, offset, &wrd); 1421201052Smarius r = wrd; 1422201052Smarius break; 1423201052Smarius default: 1424201052Smarius panic("%s: bad width", __func__); 1425201052Smarius /* NOTREACHED */ 1426201052Smarius } 1427201052Smarius 1428201052Smarius if (i) { 1429201052Smarius#ifdef FIRE_DEBUG 1430201052Smarius printf("%s: read data error reading: %d.%d.%d: 0x%x\n", 1431201052Smarius __func__, bus, slot, func, reg); 1432201052Smarius#endif 1433201052Smarius r = -1; 1434201052Smarius } 1435201052Smarius return (r); 1436201052Smarius} 1437201052Smarius 1438201052Smariusstatic void 1439201052Smariusfire_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, 1440201052Smarius uint32_t val, int width) 1441201052Smarius{ 1442201052Smarius struct fire_softc *sc; 1443201052Smarius bus_space_handle_t bh; 1444201052Smarius u_long offset = 0; 1445201052Smarius 1446201052Smarius sc = device_get_softc(dev); 1447201052Smarius if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus || 1448201052Smarius slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCIE_REGMAX) 1449201052Smarius return; 1450201052Smarius 1451201052Smarius offset = FO_CONF_OFF(bus, slot, func, reg); 1452201052Smarius bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG]; 1453201052Smarius switch (width) { 1454201052Smarius case 1: 1455201052Smarius bus_space_write_1(sc->sc_pci_cfgt, bh, offset, val); 1456201052Smarius break; 1457201052Smarius case 2: 1458201052Smarius bus_space_write_2(sc->sc_pci_cfgt, bh, offset, val); 1459201052Smarius break; 1460201052Smarius case 4: 1461201052Smarius bus_space_write_4(sc->sc_pci_cfgt, bh, offset, val); 1462201052Smarius break; 1463201052Smarius default: 1464201052Smarius panic("%s: bad width", __func__); 1465201052Smarius /* NOTREACHED */ 1466201052Smarius } 1467201052Smarius} 1468201052Smarius 1469201052Smariusstatic int 1470201052Smariusfire_route_interrupt(device_t bridge, device_t dev, int pin) 1471201052Smarius{ 1472201052Smarius struct fire_softc *sc; 1473201052Smarius struct ofw_pci_register reg; 1474201052Smarius ofw_pci_intr_t pintr, mintr; 1475201052Smarius uint8_t maskbuf[sizeof(reg) + sizeof(pintr)]; 1476201052Smarius 1477201052Smarius sc = device_get_softc(bridge); 1478201052Smarius pintr = pin; 1479201052Smarius if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo, 1480201052Smarius ®, sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr), 1481209298Snwhitehorn NULL, maskbuf) != 0) 1482201052Smarius return (mintr); 1483201052Smarius 1484201052Smarius device_printf(bridge, "could not route pin %d for device %d.%d\n", 1485201052Smarius pin, pci_get_slot(dev), pci_get_function(dev)); 1486201052Smarius return (PCI_INVALID_IRQ); 1487201052Smarius} 1488201052Smarius 1489201052Smariusstatic int 1490201052Smariusfire_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) 1491201052Smarius{ 1492201052Smarius struct fire_softc *sc; 1493201052Smarius 1494201052Smarius sc = device_get_softc(dev); 1495201052Smarius switch (which) { 1496201052Smarius case PCIB_IVAR_DOMAIN: 1497201052Smarius *result = device_get_unit(dev); 1498201052Smarius return (0); 1499201052Smarius case PCIB_IVAR_BUS: 1500201052Smarius *result = sc->sc_pci_secbus; 1501201052Smarius return (0); 1502201052Smarius } 1503201052Smarius return (ENOENT); 1504201052Smarius} 1505201052Smarius 1506201052Smariusstatic void 1507201052Smariusfire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map, 1508201052Smarius bus_dmasync_op_t op) 1509201052Smarius{ 1510201052Smarius static u_char buf[VIS_BLOCKSIZE] __aligned(VIS_BLOCKSIZE); 1511201052Smarius register_t reg, s; 1512201052Smarius 1513219785Smarius if ((map->dm_flags & DMF_LOADED) == 0) 1514201052Smarius return; 1515201052Smarius 1516219785Smarius if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1517219785Smarius s = intr_disable(); 1518219785Smarius reg = rd(fprs); 1519219785Smarius wr(fprs, reg | FPRS_FEF, 0); 1520219785Smarius __asm __volatile("stda %%f0, [%0] %1" 1521219785Smarius : : "r" (buf), "n" (ASI_BLK_COMMIT_S)); 1522219785Smarius membar(Sync); 1523219785Smarius wr(fprs, reg, 0); 1524219785Smarius intr_restore(s); 1525219785Smarius } else if ((op & BUS_DMASYNC_PREWRITE) != 0) 1526219785Smarius membar(Sync); 1527201052Smarius} 1528201052Smarius 1529201052Smariusstatic void 1530201052Smariusfire_intr_enable(void *arg) 1531201052Smarius{ 1532201052Smarius struct intr_vector *iv; 1533201052Smarius struct fire_icarg *fica; 1534201052Smarius struct fire_softc *sc; 1535201052Smarius struct pcpu *pc; 1536201052Smarius uint64_t mr; 1537201052Smarius u_int ctrl, i; 1538201052Smarius 1539201052Smarius iv = arg; 1540201052Smarius fica = iv->iv_icarg; 1541201052Smarius sc = fica->fica_sc; 1542201052Smarius mr = FO_PCI_IMAP_V; 1543201052Smarius if (sc->sc_mode == FIRE_MODE_OBERON) 1544201052Smarius mr |= (iv->iv_mid << OBERON_PCI_IMAP_T_DESTID_SHFT) & 1545201052Smarius OBERON_PCI_IMAP_T_DESTID_MASK; 1546201052Smarius else 1547201052Smarius mr |= (iv->iv_mid << FIRE_PCI_IMAP_T_JPID_SHFT) & 1548201052Smarius FIRE_PCI_IMAP_T_JPID_MASK; 1549201052Smarius /* 1550201052Smarius * Given that all mondos for the same target are required to use the 1551201052Smarius * same interrupt controller we just use the CPU ID for indexing the 1552201052Smarius * latter. 1553201052Smarius */ 1554201052Smarius ctrl = 0; 1555201052Smarius for (i = 0; i < mp_ncpus; ++i) { 1556201052Smarius pc = pcpu_find(i); 1557201052Smarius if (pc == NULL || iv->iv_mid != pc->pc_mid) 1558201052Smarius continue; 1559201052Smarius ctrl = pc->pc_cpuid % 4; 1560201052Smarius break; 1561201052Smarius } 1562201052Smarius mr |= (1ULL << ctrl) << FO_PCI_IMAP_INT_CTRL_NUM_SHFT & 1563201052Smarius FO_PCI_IMAP_INT_CTRL_NUM_MASK; 1564201052Smarius FIRE_PCI_WRITE_8(sc, fica->fica_map, mr); 1565201052Smarius} 1566201052Smarius 1567201052Smariusstatic void 1568201052Smariusfire_intr_disable(void *arg) 1569201052Smarius{ 1570201052Smarius struct intr_vector *iv; 1571201052Smarius struct fire_icarg *fica; 1572201052Smarius struct fire_softc *sc; 1573201052Smarius 1574201052Smarius iv = arg; 1575201052Smarius fica = iv->iv_icarg; 1576201052Smarius sc = fica->fica_sc; 1577201052Smarius FIRE_PCI_WRITE_8(sc, fica->fica_map, 1578201052Smarius FIRE_PCI_READ_8(sc, fica->fica_map) & ~FO_PCI_IMAP_V); 1579201052Smarius} 1580201052Smarius 1581201052Smariusstatic void 1582201052Smariusfire_intr_assign(void *arg) 1583201052Smarius{ 1584201052Smarius struct intr_vector *iv; 1585201052Smarius struct fire_icarg *fica; 1586201052Smarius struct fire_softc *sc; 1587201052Smarius uint64_t mr; 1588201052Smarius 1589201052Smarius iv = arg; 1590201052Smarius fica = iv->iv_icarg; 1591201052Smarius sc = fica->fica_sc; 1592201052Smarius mr = FIRE_PCI_READ_8(sc, fica->fica_map); 1593201052Smarius if ((mr & FO_PCI_IMAP_V) != 0) { 1594201052Smarius FIRE_PCI_WRITE_8(sc, fica->fica_map, mr & ~FO_PCI_IMAP_V); 1595201052Smarius FIRE_PCI_BARRIER(sc, fica->fica_map, 8, 1596201052Smarius BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 1597201052Smarius } 1598201052Smarius while (FIRE_PCI_READ_8(sc, fica->fica_clr) != INTCLR_IDLE) 1599201052Smarius ; 1600201052Smarius if ((mr & FO_PCI_IMAP_V) != 0) 1601201052Smarius fire_intr_enable(arg); 1602201052Smarius} 1603201052Smarius 1604201052Smariusstatic void 1605201052Smariusfire_intr_clear(void *arg) 1606201052Smarius{ 1607201052Smarius struct intr_vector *iv; 1608201052Smarius struct fire_icarg *fica; 1609201052Smarius 1610201052Smarius iv = arg; 1611201052Smarius fica = iv->iv_icarg; 1612201052Smarius FIRE_PCI_WRITE_8(fica->fica_sc, fica->fica_clr, INTCLR_IDLE); 1613201052Smarius} 1614201052Smarius 1615201052Smarius/* 1616201052Smarius * Given that the event queue implementation matches our current MD and MI 1617201052Smarius * interrupt frameworks like square pegs fit into round holes we are generous 1618201052Smarius * and use one event queue per MSI for now, which limits us to 35 MSIs/MSI-Xs 1619201052Smarius * per Host-PCIe-bridge (we use one event queue for the PCIe error messages). 1620201052Smarius * This seems tolerable as long as most devices just use one MSI/MSI-X anyway. 1621201052Smarius * Adding knowledge about MSIs/MSI-Xs to the MD interrupt code should allow us 1622201052Smarius * to decouple the 1:1 mapping at the cost of no longer being able to bind 1623201052Smarius * MSIs/MSI-Xs to specific CPUs as we currently have no reliable way to 1624201052Smarius * quiesce a device while we move its MSIs/MSI-Xs to another event queue. 1625201052Smarius */ 1626201052Smarius 1627201052Smariusstatic int 1628202003Smariusfire_alloc_msi(device_t dev, device_t child, int count, int maxcount __unused, 1629201052Smarius int *irqs) 1630201052Smarius{ 1631201052Smarius struct fire_softc *sc; 1632201052Smarius u_int i, j, msiqrun; 1633201052Smarius 1634201052Smarius if (powerof2(count) == 0 || count > 32) 1635201052Smarius return (EINVAL); 1636201052Smarius 1637201052Smarius sc = device_get_softc(dev); 1638201052Smarius mtx_lock(&sc->sc_msi_mtx); 1639201052Smarius msiqrun = 0; 1640201052Smarius for (i = 0; i < sc->sc_msiq_count; i++) { 1641201052Smarius for (j = i; j < i + count; j++) { 1642201052Smarius if (isclr(sc->sc_msiq_bitmap, j) == 0) 1643201052Smarius break; 1644201052Smarius } 1645201052Smarius if (j == i + count) { 1646201052Smarius msiqrun = i; 1647201052Smarius break; 1648201052Smarius } 1649201052Smarius } 1650201052Smarius if (i == sc->sc_msiq_count) { 1651201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1652201052Smarius return (ENXIO); 1653201052Smarius } 1654202003Smarius for (i = 0; i + count < sc->sc_msi_count; i += count) { 1655202003Smarius for (j = i; j < i + count; j++) 1656201052Smarius if (isclr(sc->sc_msi_bitmap, j) == 0) 1657201052Smarius break; 1658202003Smarius if (j == i + count) { 1659201052Smarius for (j = 0; j < count; j++) { 1660201052Smarius setbit(sc->sc_msiq_bitmap, msiqrun + j); 1661201052Smarius setbit(sc->sc_msi_bitmap, i + j); 1662201052Smarius sc->sc_msi_msiq_table[i + j] = msiqrun + j; 1663201052Smarius irqs[j] = sc->sc_msi_first + i + j; 1664201052Smarius } 1665201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1666201052Smarius return (0); 1667201052Smarius } 1668201052Smarius } 1669201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1670201052Smarius return (ENXIO); 1671201052Smarius} 1672201052Smarius 1673201052Smariusstatic int 1674201052Smariusfire_release_msi(device_t dev, device_t child, int count, int *irqs) 1675201052Smarius{ 1676201052Smarius struct fire_softc *sc; 1677201052Smarius u_int i; 1678201052Smarius 1679201052Smarius sc = device_get_softc(dev); 1680201052Smarius mtx_lock(&sc->sc_msi_mtx); 1681201052Smarius for (i = 0; i < count; i++) { 1682201052Smarius clrbit(sc->sc_msiq_bitmap, 1683201052Smarius sc->sc_msi_msiq_table[irqs[i] - sc->sc_msi_first]); 1684201052Smarius clrbit(sc->sc_msi_bitmap, irqs[i] - sc->sc_msi_first); 1685201052Smarius } 1686201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1687201052Smarius return (0); 1688201052Smarius} 1689201052Smarius 1690201052Smariusstatic int 1691201052Smariusfire_alloc_msix(device_t dev, device_t child, int *irq) 1692201052Smarius{ 1693201052Smarius struct fire_softc *sc; 1694201052Smarius u_int i, msiq; 1695201052Smarius 1696201052Smarius sc = device_get_softc(dev); 1697201052Smarius if ((sc->sc_flags & FIRE_MSIX) == 0) 1698201052Smarius return (ENXIO); 1699201052Smarius mtx_lock(&sc->sc_msi_mtx); 1700201052Smarius msiq = 0; 1701201052Smarius for (i = 0; i < sc->sc_msiq_count; i++) { 1702201052Smarius if (isclr(sc->sc_msiq_bitmap, i) != 0) { 1703201052Smarius msiq = i; 1704201052Smarius break; 1705201052Smarius } 1706201052Smarius } 1707201052Smarius if (i == sc->sc_msiq_count) { 1708201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1709201052Smarius return (ENXIO); 1710201052Smarius } 1711201052Smarius for (i = sc->sc_msi_count - 1; i >= 0; i--) { 1712201052Smarius if (isclr(sc->sc_msi_bitmap, i) != 0) { 1713201052Smarius setbit(sc->sc_msiq_bitmap, msiq); 1714201052Smarius setbit(sc->sc_msi_bitmap, i); 1715201052Smarius sc->sc_msi_msiq_table[i] = msiq; 1716201052Smarius *irq = sc->sc_msi_first + i; 1717201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1718201052Smarius return (0); 1719201052Smarius } 1720201052Smarius } 1721201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1722201052Smarius return (ENXIO); 1723201052Smarius} 1724201052Smarius 1725201052Smariusstatic int 1726201052Smariusfire_release_msix(device_t dev, device_t child, int irq) 1727201052Smarius{ 1728201052Smarius struct fire_softc *sc; 1729201052Smarius 1730201052Smarius sc = device_get_softc(dev); 1731201052Smarius if ((sc->sc_flags & FIRE_MSIX) == 0) 1732201052Smarius return (ENXIO); 1733201052Smarius mtx_lock(&sc->sc_msi_mtx); 1734201052Smarius clrbit(sc->sc_msiq_bitmap, 1735201052Smarius sc->sc_msi_msiq_table[irq - sc->sc_msi_first]); 1736201052Smarius clrbit(sc->sc_msi_bitmap, irq - sc->sc_msi_first); 1737201052Smarius mtx_unlock(&sc->sc_msi_mtx); 1738201052Smarius return (0); 1739201052Smarius} 1740201052Smarius 1741201052Smariusstatic int 1742201052Smariusfire_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, 1743201052Smarius uint32_t *data) 1744201052Smarius{ 1745201052Smarius struct fire_softc *sc; 1746201052Smarius struct pci_devinfo *dinfo; 1747201052Smarius 1748201052Smarius sc = device_get_softc(dev); 1749201052Smarius dinfo = device_get_ivars(child); 1750201052Smarius if (dinfo->cfg.msi.msi_alloc > 0) { 1751201052Smarius if ((irq & ~sc->sc_msi_data_mask) != 0) { 1752201052Smarius device_printf(dev, "invalid MSI 0x%x\n", irq); 1753201052Smarius return (EINVAL); 1754201052Smarius } 1755201052Smarius } else { 1756201052Smarius if ((sc->sc_flags & FIRE_MSIX) == 0) 1757201052Smarius return (ENXIO); 1758201052Smarius if (fls(irq) > sc->sc_msix_data_width) { 1759201052Smarius device_printf(dev, "invalid MSI-X 0x%x\n", irq); 1760201052Smarius return (EINVAL); 1761201052Smarius } 1762201052Smarius } 1763201052Smarius if (dinfo->cfg.msi.msi_alloc > 0 && 1764201052Smarius (dinfo->cfg.msi.msi_ctrl & PCIM_MSICTRL_64BIT) == 0) 1765201052Smarius *addr = sc->sc_msi_addr32; 1766201052Smarius else 1767201052Smarius *addr = sc->sc_msi_addr64; 1768201052Smarius *data = irq; 1769201052Smarius return (0); 1770201052Smarius} 1771201052Smarius 1772201052Smariusstatic void 1773201052Smariusfire_msiq_handler(void *cookie) 1774201052Smarius{ 1775201052Smarius struct intr_vector *iv; 1776201052Smarius struct fire_msiqarg *fmqa; 1777201052Smarius 1778201052Smarius iv = cookie; 1779201052Smarius fmqa = iv->iv_icarg; 1780201052Smarius /* 1781201052Smarius * Note that since fire_intr_clear() will clear the event queue 1782202003Smarius * interrupt after the handler associated with the MSI [sic] has 1783202003Smarius * been executed we have to protect the access to the event queue as 1784202003Smarius * otherwise nested event queue interrupts cause corruption of the 1785201052Smarius * event queue on MP machines. Obviously especially when abandoning 1786201052Smarius * the 1:1 mapping it would be better to not clear the event queue 1787202003Smarius * interrupt after each handler invocation but only once when the 1788202003Smarius * outstanding MSIs have been processed but unfortunately that 1789201052Smarius * doesn't work well and leads to interrupt storms with controllers/ 1790202003Smarius * drivers which don't mask interrupts while the handler is executed. 1791202003Smarius * Maybe delaying clearing the MSI until after the handler has been 1792202003Smarius * executed could be used to work around this but that's not the 1793202003Smarius * intended usage and might in turn cause lost MSIs. 1794201052Smarius */ 1795201052Smarius mtx_lock_spin(&fmqa->fmqa_mtx); 1796202003Smarius fire_msiq_common(iv, fmqa); 1797202003Smarius mtx_unlock_spin(&fmqa->fmqa_mtx); 1798202003Smarius} 1799202003Smarius 1800202003Smariusstatic void 1801202003Smariusfire_msiq_filter(void *cookie) 1802202003Smarius{ 1803202003Smarius struct intr_vector *iv; 1804202003Smarius struct fire_msiqarg *fmqa; 1805202003Smarius 1806202003Smarius iv = cookie; 1807202003Smarius fmqa = iv->iv_icarg; 1808202003Smarius /* 1809202003Smarius * For filters we don't use fire_intr_clear() since it would clear 1810202003Smarius * the event queue interrupt while we're still processing the event 1811202003Smarius * queue as filters and associated post-filter handler are executed 1812202003Smarius * directly, which in turn would lead to lost MSIs. So we clear the 1813202003Smarius * event queue interrupt only once after processing the event queue. 1814202003Smarius * Given that this still guarantees the filters to not be executed 1815202003Smarius * concurrently and no other CPU can clear the event queue interrupt 1816202003Smarius * while the event queue is still processed, we don't even need to 1817202003Smarius * interlock the access to the event queue in this case. 1818202003Smarius */ 1819202003Smarius critical_enter(); 1820202003Smarius fire_msiq_common(iv, fmqa); 1821202003Smarius FIRE_PCI_WRITE_8(fmqa->fmqa_fica.fica_sc, fmqa->fmqa_fica.fica_clr, 1822202003Smarius INTCLR_IDLE); 1823202003Smarius critical_exit(); 1824202003Smarius} 1825202003Smarius 1826202003Smariusstatic inline void 1827202003Smariusfire_msiq_common(struct intr_vector *iv, struct fire_msiqarg *fmqa) 1828202003Smarius{ 1829202003Smarius struct fire_softc *sc; 1830202003Smarius struct fo_msiq_record *qrec; 1831202003Smarius device_t dev; 1832202003Smarius uint64_t word0; 1833202003Smarius u_int head, msi, msiq; 1834202003Smarius 1835202003Smarius sc = fmqa->fmqa_fica.fica_sc; 1836202003Smarius dev = sc->sc_dev; 1837202003Smarius msiq = fmqa->fmqa_msiq; 1838201052Smarius head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >> 1839201052Smarius FO_PCI_EQ_HD_SHFT; 1840201052Smarius qrec = &fmqa->fmqa_base[head]; 1841201052Smarius word0 = qrec->fomqr_word0; 1842201052Smarius for (;;) { 1843203094Smarius if (__predict_false((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0)) 1844203094Smarius break; 1845201052Smarius KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSI64) != 0 || 1846201052Smarius (word0 & FO_MQR_WORD0_FMT_TYPE_MSI32) != 0, 1847201052Smarius ("%s: received non-MSI/MSI-X message in event queue %d " 1848201052Smarius "(word0 %#llx)", device_get_nameunit(dev), msiq, 1849201052Smarius (unsigned long long)word0)); 1850201052Smarius msi = (word0 & FO_MQR_WORD0_DATA0_MASK) >> 1851201052Smarius FO_MQR_WORD0_DATA0_SHFT; 1852201052Smarius /* 1853201052Smarius * Sanity check the MSI/MSI-X as long as we use a 1:1 mapping. 1854201052Smarius */ 1855201052Smarius KASSERT(msi == fmqa->fmqa_msi, 1856201052Smarius ("%s: received non-matching MSI/MSI-X in event queue %d " 1857201052Smarius "(%d versus %d)", device_get_nameunit(dev), msiq, msi, 1858201052Smarius fmqa->fmqa_msi)); 1859201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + (msi << 3), 1860201052Smarius FO_PCI_MSI_CLR_EQWR_N); 1861201052Smarius if (__predict_false(intr_event_handle(iv->iv_event, 1862201052Smarius NULL) != 0)) 1863201052Smarius printf("stray MSI/MSI-X in event queue %d\n", msiq); 1864201052Smarius qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK; 1865201052Smarius head = (head + 1) % sc->sc_msiq_size; 1866201052Smarius qrec = &fmqa->fmqa_base[head]; 1867201052Smarius word0 = qrec->fomqr_word0; 1868201052Smarius } 1869201052Smarius FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) << 1870201052Smarius FO_PCI_EQ_HD_SHFT); 1871201052Smarius if (__predict_false((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) & 1872201052Smarius FO_PCI_EQ_TL_OVERR) != 0)) { 1873201052Smarius device_printf(dev, "event queue %d overflow\n", msiq); 1874201052Smarius msiq <<= 3; 1875201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, 1876201052Smarius FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) | 1877201052Smarius FO_PCI_EQ_CTRL_CLR_COVERR); 1878201052Smarius } 1879201052Smarius} 1880201052Smarius 1881201052Smariusstatic int 1882201052Smariusfire_setup_intr(device_t dev, device_t child, struct resource *ires, 1883201052Smarius int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, 1884201052Smarius void **cookiep) 1885201052Smarius{ 1886201052Smarius struct fire_softc *sc; 1887202023Smarius struct fire_msiqarg *fmqa; 1888201052Smarius u_long vec; 1889201052Smarius int error; 1890201052Smarius u_int msi, msiq; 1891201052Smarius 1892201052Smarius sc = device_get_softc(dev); 1893201052Smarius /* 1894201052Smarius * XXX this assumes that a device only has one INTx, while in fact 1895201052Smarius * Cassini+ and Saturn can use all four the firmware has assigned 1896201052Smarius * to them, but so does pci(4). 1897201052Smarius */ 1898201052Smarius if (rman_get_rid(ires) != 0) { 1899201052Smarius msi = rman_get_start(ires); 1900201052Smarius msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first]; 1901201052Smarius vec = INTMAP_VEC(sc->sc_ign, sc->sc_msiq_ino_first + msiq); 1902201052Smarius msiq += sc->sc_msiq_first; 1903201052Smarius if (intr_vectors[vec].iv_ic != &fire_ic) { 1904201052Smarius device_printf(dev, 1905201052Smarius "invalid interrupt controller for vector 0x%lx\n", 1906201052Smarius vec); 1907201052Smarius return (EINVAL); 1908201052Smarius } 1909201052Smarius /* 1910201052Smarius * The MD interrupt code needs the vector rather than the MSI. 1911201052Smarius */ 1912201052Smarius rman_set_start(ires, vec); 1913201052Smarius rman_set_end(ires, vec); 1914201052Smarius error = bus_generic_setup_intr(dev, child, ires, flags, filt, 1915201052Smarius intr, arg, cookiep); 1916201052Smarius rman_set_start(ires, msi); 1917201052Smarius rman_set_end(ires, msi); 1918202003Smarius if (error != 0) 1919202003Smarius return (error); 1920202023Smarius fmqa = intr_vectors[vec].iv_icarg; 1921202003Smarius /* 1922202003Smarius * XXX inject our event queue handler. 1923202003Smarius */ 1924202003Smarius if (filt != NULL) { 1925202003Smarius intr_vectors[vec].iv_func = fire_msiq_filter; 1926202003Smarius intr_vectors[vec].iv_ic = &fire_msiqc_filter; 1927202023Smarius /* 1928202023Smarius * Ensure the event queue interrupt is cleared, it 1929202023Smarius * might have triggered before. Given we supply NULL 1930202023Smarius * as ic_clear, inthand_add() won't do this for us. 1931202023Smarius */ 1932202023Smarius FIRE_PCI_WRITE_8(sc, fmqa->fmqa_fica.fica_clr, 1933202023Smarius INTCLR_IDLE); 1934202003Smarius } else 1935201052Smarius intr_vectors[vec].iv_func = fire_msiq_handler; 1936202003Smarius /* Record the MSI/MSI-X as long as we we use a 1:1 mapping. */ 1937202023Smarius fmqa->fmqa_msi = msi; 1938202003Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (msiq << 3), 1939202003Smarius FO_PCI_EQ_CTRL_SET_EN); 1940202003Smarius msi <<= 3; 1941202003Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, 1942202003Smarius (FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) & 1943202003Smarius ~FO_PCI_MSI_MAP_EQNUM_MASK) | 1944202003Smarius ((msiq << FO_PCI_MSI_MAP_EQNUM_SHFT) & 1945202003Smarius FO_PCI_MSI_MAP_EQNUM_MASK)); 1946202003Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + msi, 1947202003Smarius FO_PCI_MSI_CLR_EQWR_N); 1948202003Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, 1949202003Smarius FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) | 1950202003Smarius FO_PCI_MSI_MAP_V); 1951201052Smarius return (error); 1952201052Smarius } 1953201052Smarius 1954201052Smarius /* 1955201052Smarius * Make sure the vector is fully specified and we registered 1956201052Smarius * our interrupt controller for it. 1957201052Smarius */ 1958201052Smarius vec = rman_get_start(ires); 1959201052Smarius if (INTIGN(vec) != sc->sc_ign) { 1960201052Smarius device_printf(dev, "invalid interrupt vector 0x%lx\n", vec); 1961201052Smarius return (EINVAL); 1962201052Smarius } 1963201052Smarius if (intr_vectors[vec].iv_ic != &fire_ic) { 1964201052Smarius device_printf(dev, 1965201052Smarius "invalid interrupt controller for vector 0x%lx\n", vec); 1966201052Smarius return (EINVAL); 1967201052Smarius } 1968201052Smarius return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr, 1969201052Smarius arg, cookiep)); 1970201052Smarius} 1971201052Smarius 1972201052Smariusstatic int 1973201052Smariusfire_teardown_intr(device_t dev, device_t child, struct resource *ires, 1974201052Smarius void *cookie) 1975201052Smarius{ 1976201052Smarius struct fire_softc *sc; 1977201052Smarius u_long vec; 1978201052Smarius int error; 1979201052Smarius u_int msi, msiq; 1980201052Smarius 1981201052Smarius sc = device_get_softc(dev); 1982201052Smarius if (rman_get_rid(ires) != 0) { 1983201052Smarius msi = rman_get_start(ires); 1984201052Smarius msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first]; 1985201052Smarius vec = INTMAP_VEC(sc->sc_ign, msiq + sc->sc_msiq_ino_first); 1986201052Smarius msiq += sc->sc_msiq_first; 1987201052Smarius msi <<= 3; 1988201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, 1989201052Smarius FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) & 1990201052Smarius ~FO_PCI_MSI_MAP_V); 1991201052Smarius msiq <<= 3; 1992201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, 1993201052Smarius FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I | 1994201052Smarius FO_PCI_EQ_CTRL_CLR_DIS); 1995201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_TL_BASE + msiq, 1996201052Smarius (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK); 1997201052Smarius FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_HD_BASE + msiq, 1998201052Smarius (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK); 1999202003Smarius intr_vectors[vec].iv_ic = &fire_ic; 2000201052Smarius /* 2001201052Smarius * The MD interrupt code needs the vector rather than the MSI. 2002201052Smarius */ 2003201052Smarius rman_set_start(ires, vec); 2004201052Smarius rman_set_end(ires, vec); 2005201052Smarius error = bus_generic_teardown_intr(dev, child, ires, cookie); 2006202003Smarius msi >>= 3; 2007201052Smarius rman_set_start(ires, msi); 2008202003Smarius rman_set_end(ires, msi); 2009201052Smarius return (error); 2010201052Smarius } 2011201052Smarius return (bus_generic_teardown_intr(dev, child, ires, cookie)); 2012201052Smarius} 2013201052Smarius 2014201052Smariusstatic struct resource * 2015201052Smariusfire_alloc_resource(device_t bus, device_t child, int type, int *rid, 2016201052Smarius u_long start, u_long end, u_long count, u_int flags) 2017201052Smarius{ 2018201052Smarius struct fire_softc *sc; 2019201052Smarius struct resource *rv; 2020201052Smarius struct rman *rm; 2021201052Smarius 2022201052Smarius sc = device_get_softc(bus); 2023225931Smarius switch (type) { 2024225931Smarius case SYS_RES_IRQ: 2025201052Smarius /* 2026201052Smarius * XXX: Don't accept blank ranges for now, only single 2027201052Smarius * interrupts. The other case should not happen with 2028201052Smarius * the MI PCI code... 2029201052Smarius * XXX: This may return a resource that is out of the 2030201052Smarius * range that was specified. Is this correct...? 2031201052Smarius */ 2032201052Smarius if (start != end) 2033201052Smarius panic("%s: XXX: interrupt range", __func__); 2034201052Smarius if (*rid == 0) 2035201052Smarius start = end = INTMAP_VEC(sc->sc_ign, end); 2036225931Smarius return (bus_generic_alloc_resource(bus, child, type, rid, 2037225931Smarius start, end, count, flags)); 2038201052Smarius case SYS_RES_MEMORY: 2039201052Smarius rm = &sc->sc_pci_mem_rman; 2040201052Smarius break; 2041201052Smarius case SYS_RES_IOPORT: 2042201052Smarius rm = &sc->sc_pci_io_rman; 2043201052Smarius break; 2044201052Smarius default: 2045201052Smarius return (NULL); 2046201052Smarius } 2047201052Smarius 2048225931Smarius rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE, 2049225931Smarius child); 2050201052Smarius if (rv == NULL) 2051201052Smarius return (NULL); 2052201052Smarius rman_set_rid(rv, *rid); 2053201052Smarius 2054225931Smarius if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type, 2055225931Smarius *rid, rv) != 0) { 2056225931Smarius rman_release_resource(rv); 2057225931Smarius return (NULL); 2058201052Smarius } 2059201052Smarius return (rv); 2060201052Smarius} 2061201052Smarius 2062201052Smariusstatic int 2063201052Smariusfire_activate_resource(device_t bus, device_t child, int type, int rid, 2064201052Smarius struct resource *r) 2065201052Smarius{ 2066225931Smarius struct fire_softc *sc; 2067225931Smarius struct bus_space_tag *tag; 2068201052Smarius 2069225931Smarius sc = device_get_softc(bus); 2070225931Smarius switch (type) { 2071225931Smarius case SYS_RES_IRQ: 2072225931Smarius return (bus_generic_activate_resource(bus, child, type, rid, 2073225931Smarius r)); 2074225931Smarius case SYS_RES_MEMORY: 2075225931Smarius tag = sparc64_alloc_bus_tag(r, rman_get_bustag( 2076225931Smarius sc->sc_mem_res[FIRE_PCI]), PCI_MEMORY_BUS_SPACE, NULL); 2077225931Smarius if (tag == NULL) 2078225931Smarius return (ENOMEM); 2079225931Smarius rman_set_bustag(r, tag); 2080225931Smarius rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] + 2081225931Smarius rman_get_start(r)); 2082225931Smarius break; 2083225931Smarius case SYS_RES_IOPORT: 2084225931Smarius rman_set_bustag(r, sc->sc_pci_iot); 2085225931Smarius rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] + 2086225931Smarius rman_get_start(r)); 2087225931Smarius break; 2088201052Smarius } 2089201052Smarius return (rman_activate_resource(r)); 2090201052Smarius} 2091201052Smarius 2092201052Smariusstatic int 2093225931Smariusfire_adjust_resource(device_t bus, device_t child, int type, 2094225931Smarius struct resource *r, u_long start, u_long end) 2095201052Smarius{ 2096225931Smarius struct fire_softc *sc; 2097225931Smarius struct rman *rm; 2098201052Smarius 2099225931Smarius sc = device_get_softc(bus); 2100225931Smarius switch (type) { 2101225931Smarius case SYS_RES_IRQ: 2102225931Smarius return (bus_generic_adjust_resource(bus, child, type, r, 2103225931Smarius start, end)); 2104225931Smarius case SYS_RES_MEMORY: 2105225931Smarius rm = &sc->sc_pci_mem_rman; 2106225931Smarius break; 2107225931Smarius case SYS_RES_IOPORT: 2108225931Smarius rm = &sc->sc_pci_io_rman; 2109225931Smarius break; 2110225931Smarius default: 2111225931Smarius return (EINVAL); 2112201052Smarius } 2113225931Smarius if (rman_is_region_manager(r, rm) == 0) 2114225931Smarius return (EINVAL); 2115225931Smarius return (rman_adjust_resource(r, start, end)); 2116201052Smarius} 2117201052Smarius 2118201052Smariusstatic bus_dma_tag_t 2119219785Smariusfire_get_dma_tag(device_t bus, device_t child __unused) 2120201052Smarius{ 2121201052Smarius struct fire_softc *sc; 2122201052Smarius 2123201052Smarius sc = device_get_softc(bus); 2124201052Smarius return (sc->sc_pci_dmat); 2125201052Smarius} 2126201052Smarius 2127201052Smariusstatic phandle_t 2128219785Smariusfire_get_node(device_t bus, device_t child __unused) 2129201052Smarius{ 2130201052Smarius struct fire_softc *sc; 2131201052Smarius 2132201052Smarius sc = device_get_softc(bus); 2133201052Smarius /* We only have one child, the PCI bus, which needs our own node. */ 2134201052Smarius return (sc->sc_node); 2135201052Smarius} 2136201052Smarius 2137201052Smariusstatic u_int 2138201052Smariusfire_get_timecount(struct timecounter *tc) 2139201052Smarius{ 2140201052Smarius struct fire_softc *sc; 2141201052Smarius 2142201052Smarius sc = tc->tc_priv; 2143201052Smarius return (FIRE_CTRL_READ_8(sc, FO_XBC_PRF_CNT0) & TC_COUNTER_MAX_MASK); 2144201052Smarius} 2145