fire.c revision 266020
133965Sjdp/*- 2218822Sdim * Copyright (c) 1999, 2000 Matthew R. Green 3218822Sdim * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org> 433965Sjdp * Copyright (c) 2009 by Marius Strobl <marius@FreeBSD.org> 533965Sjdp * All rights reserved. 633965Sjdp * 733965Sjdp * Redistribution and use in source and binary forms, with or without 833965Sjdp * modification, are permitted provided that the following conditions 933965Sjdp * are met: 1033965Sjdp * 1. Redistributions of source code must retain the above copyright 1133965Sjdp * notice, this list of conditions and the following disclaimer. 1233965Sjdp * 2. Redistributions in binary form must reproduce the above copyright 1333965Sjdp * notice, this list of conditions and the following disclaimer in the 1433965Sjdp * documentation and/or other materials provided with the distribution. 1533965Sjdp * 3. The name of the author may not be used to endorse or promote products 1633965Sjdp * derived from this software without specific prior written permission. 1733965Sjdp * 1833965Sjdp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1933965Sjdp * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20218822Sdim * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21218822Sdim * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2233965Sjdp * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 2333965Sjdp * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 2433965Sjdp * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 2533965Sjdp * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 2633965Sjdp * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2733965Sjdp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28218822Sdim * SUCH DAMAGE. 2933965Sjdp * 3033965Sjdp * from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp 3189857Sobrien * from: FreeBSD: psycho.c 183152 2008-09-18 19:45:22Z marius 3233965Sjdp */ 3333965Sjdp 3433965Sjdp#include <sys/cdefs.h> 3577298Sobrien__FBSDID("$FreeBSD: stable/10/sys/sparc64/pci/fire.c 266020 2014-05-14 14:17:51Z ian $"); 3633965Sjdp 3733965Sjdp/* 3833965Sjdp * Driver for `Fire' JBus to PCI Express and `Oberon' Uranus to PCI Express 3933965Sjdp * bridges 4033965Sjdp */ 4133965Sjdp 4233965Sjdp#include "opt_fire.h" 4333965Sjdp#include "opt_ofw_pci.h" 4433965Sjdp 4533965Sjdp#include <sys/param.h> 4633965Sjdp#include <sys/systm.h> 4733965Sjdp#include <sys/bus.h> 4833965Sjdp#include <sys/interrupt.h> 4933965Sjdp#include <sys/kernel.h> 50130561Sobrien#include <sys/lock.h> 51130561Sobrien#include <sys/malloc.h> 5233965Sjdp#include <sys/module.h> 5333965Sjdp#include <sys/mutex.h> 5433965Sjdp#include <sys/pciio.h> 5533965Sjdp#include <sys/pcpu.h> 5633965Sjdp#include <sys/rman.h> 5733965Sjdp#include <sys/smp.h> 5833965Sjdp#include <sys/sysctl.h> 5933965Sjdp#include <sys/timetc.h> 6033965Sjdp 6133965Sjdp#include <dev/ofw/ofw_bus.h> 6233965Sjdp#include <dev/ofw/ofw_pci.h> 6333965Sjdp#include <dev/ofw/openfirm.h> 6433965Sjdp 6533965Sjdp#include <vm/vm.h> 6633965Sjdp#include <vm/pmap.h> 6733965Sjdp 6833965Sjdp#include <machine/bus.h> 6933965Sjdp#include <machine/bus_common.h> 7033965Sjdp#include <machine/bus_private.h> 71130561Sobrien#include <machine/fsr.h> 7233965Sjdp#include <machine/iommureg.h> 7333965Sjdp#include <machine/iommuvar.h> 7433965Sjdp#include <machine/pmap.h> 7533965Sjdp#include <machine/resource.h> 7633965Sjdp 7733965Sjdp#include <dev/pci/pcireg.h> 7833965Sjdp#include <dev/pci/pcivar.h> 79130561Sobrien 8033965Sjdp#include <sparc64/pci/ofw_pci.h> 8133965Sjdp#include <sparc64/pci/firereg.h> 8233965Sjdp#include <sparc64/pci/firevar.h> 8333965Sjdp 8433965Sjdp#include "pcib_if.h" 8533965Sjdp 8633965Sjdpstruct fire_msiqarg; 8733965Sjdp 8833965Sjdpstatic const struct fire_desc *fire_get_desc(device_t dev); 8933965Sjdpstatic void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map, 9033965Sjdp bus_dmasync_op_t op); 9133965Sjdpstatic int fire_get_intrmap(struct fire_softc *sc, u_int ino, 9233965Sjdp bus_addr_t *intrmapptr, bus_addr_t *intrclrptr); 9333965Sjdpstatic void fire_intr_assign(void *arg); 9433965Sjdpstatic void fire_intr_clear(void *arg); 9533965Sjdpstatic void fire_intr_disable(void *arg); 9660484Sobrienstatic void fire_intr_enable(void *arg); 9760484Sobrienstatic int fire_intr_register(struct fire_softc *sc, u_int ino); 98130561Sobrienstatic inline void fire_msiq_common(struct intr_vector *iv, 9933965Sjdp struct fire_msiqarg *fmqa); 10033965Sjdpstatic void fire_msiq_filter(void *cookie); 10133965Sjdpstatic void fire_msiq_handler(void *cookie); 10233965Sjdpstatic void fire_set_intr(struct fire_softc *sc, u_int index, u_int ino, 10333965Sjdp driver_filter_t handler, void *arg); 10433965Sjdpstatic timecounter_get_t fire_get_timecount; 10533965Sjdp 10633965Sjdp/* Interrupt handlers */ 10733965Sjdpstatic driver_filter_t fire_dmc_pec; 10833965Sjdpstatic driver_filter_t fire_pcie; 10933965Sjdpstatic driver_filter_t fire_xcb; 11033965Sjdp 11133965Sjdp/* 11233965Sjdp * Methods 11333965Sjdp */ 11433965Sjdpstatic bus_activate_resource_t fire_activate_resource; 11533965Sjdpstatic bus_adjust_resource_t fire_adjust_resource; 11633965Sjdpstatic pcib_alloc_msi_t fire_alloc_msi; 11733965Sjdpstatic pcib_alloc_msix_t fire_alloc_msix; 11833965Sjdpstatic bus_alloc_resource_t fire_alloc_resource; 11933965Sjdpstatic device_attach_t fire_attach; 12033965Sjdpstatic bus_get_dma_tag_t fire_get_dma_tag; 12133965Sjdpstatic ofw_bus_get_node_t fire_get_node; 12233965Sjdpstatic pcib_map_msi_t fire_map_msi; 12333965Sjdpstatic pcib_maxslots_t fire_maxslots; 12433965Sjdpstatic device_probe_t fire_probe; 12533965Sjdpstatic pcib_read_config_t fire_read_config; 12633965Sjdpstatic bus_read_ivar_t fire_read_ivar; 12733965Sjdpstatic pcib_release_msi_t fire_release_msi; 12833965Sjdpstatic pcib_release_msix_t fire_release_msix; 12933965Sjdpstatic pcib_route_interrupt_t fire_route_interrupt; 13033965Sjdpstatic bus_setup_intr_t fire_setup_intr; 13133965Sjdpstatic bus_teardown_intr_t fire_teardown_intr; 13233965Sjdpstatic pcib_write_config_t fire_write_config; 13333965Sjdp 13433965Sjdpstatic device_method_t fire_methods[] = { 13533965Sjdp /* Device interface */ 13633965Sjdp DEVMETHOD(device_probe, fire_probe), 13733965Sjdp DEVMETHOD(device_attach, fire_attach), 13833965Sjdp DEVMETHOD(device_shutdown, bus_generic_shutdown), 13933965Sjdp DEVMETHOD(device_suspend, bus_generic_suspend), 14033965Sjdp DEVMETHOD(device_resume, bus_generic_resume), 14133965Sjdp 14233965Sjdp /* Bus interface */ 14333965Sjdp DEVMETHOD(bus_read_ivar, fire_read_ivar), 14433965Sjdp DEVMETHOD(bus_setup_intr, fire_setup_intr), 14533965Sjdp DEVMETHOD(bus_teardown_intr, fire_teardown_intr), 14633965Sjdp DEVMETHOD(bus_alloc_resource, fire_alloc_resource), 14733965Sjdp DEVMETHOD(bus_activate_resource, fire_activate_resource), 148130561Sobrien DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), 149130561Sobrien DEVMETHOD(bus_adjust_resource, fire_adjust_resource), 150130561Sobrien DEVMETHOD(bus_release_resource, bus_generic_release_resource), 151130561Sobrien DEVMETHOD(bus_get_dma_tag, fire_get_dma_tag), 152130561Sobrien 153130561Sobrien /* pcib interface */ 15433965Sjdp DEVMETHOD(pcib_maxslots, fire_maxslots), 155130561Sobrien DEVMETHOD(pcib_read_config, fire_read_config), 156130561Sobrien DEVMETHOD(pcib_write_config, fire_write_config), 15733965Sjdp DEVMETHOD(pcib_route_interrupt, fire_route_interrupt), 158130561Sobrien DEVMETHOD(pcib_alloc_msi, fire_alloc_msi), 159130561Sobrien DEVMETHOD(pcib_release_msi, fire_release_msi), 160130561Sobrien DEVMETHOD(pcib_alloc_msix, fire_alloc_msix), 161130561Sobrien DEVMETHOD(pcib_release_msix, fire_release_msix), 16233965Sjdp DEVMETHOD(pcib_map_msi, fire_map_msi), 163130561Sobrien 164130561Sobrien /* ofw_bus interface */ 165130561Sobrien DEVMETHOD(ofw_bus_get_node, fire_get_node), 166130561Sobrien 167130561Sobrien DEVMETHOD_END 168130561Sobrien}; 169130561Sobrien 170130561Sobrienstatic devclass_t fire_devclass; 171130561Sobrien 172130561SobrienDEFINE_CLASS_0(pcib, fire_driver, fire_methods, sizeof(struct fire_softc)); 173130561SobrienEARLY_DRIVER_MODULE(fire, nexus, fire_driver, fire_devclass, 0, 0, 174130561Sobrien BUS_PASS_BUS); 175130561SobrienMODULE_DEPEND(fire, nexus, 1, 1, 1); 176130561Sobrien 17733965Sjdpstatic const struct intr_controller fire_ic = { 178130561Sobrien fire_intr_enable, 179130561Sobrien fire_intr_disable, 180130561Sobrien fire_intr_assign, 181130561Sobrien fire_intr_clear 182130561Sobrien}; 18333965Sjdp 184130561Sobrienstruct fire_icarg { 185130561Sobrien struct fire_softc *fica_sc; 186130561Sobrien bus_addr_t fica_map; 187130561Sobrien bus_addr_t fica_clr; 188130561Sobrien}; 189130561Sobrien 190130561Sobrienstatic const struct intr_controller fire_msiqc_filter = { 191130561Sobrien fire_intr_enable, 192130561Sobrien fire_intr_disable, 193130561Sobrien fire_intr_assign, 194130561Sobrien NULL 195130561Sobrien}; 19633965Sjdp 197130561Sobrienstruct fire_msiqarg { 19833965Sjdp struct fire_icarg fmqa_fica; 199130561Sobrien struct mtx fmqa_mtx; 20033965Sjdp struct fo_msiq_record *fmqa_base; 201130561Sobrien uint64_t fmqa_head; 202130561Sobrien uint64_t fmqa_tail; 203130561Sobrien uint32_t fmqa_msiq; 204218822Sdim uint32_t fmqa_msi; 205218822Sdim}; 206130561Sobrien 207130561Sobrien#define FIRE_PERF_CNT_QLTY 100 208130561Sobrien 20933965Sjdp#define FIRE_SPC_BARRIER(spc, sc, offs, len, flags) \ 21033965Sjdp bus_barrier((sc)->sc_mem_res[(spc)], (offs), (len), (flags)) 21133965Sjdp#define FIRE_SPC_READ_8(spc, sc, offs) \ 21233965Sjdp bus_read_8((sc)->sc_mem_res[(spc)], (offs)) 213130561Sobrien#define FIRE_SPC_WRITE_8(spc, sc, offs, v) \ 21433965Sjdp bus_write_8((sc)->sc_mem_res[(spc)], (offs), (v)) 21533965Sjdp 21633965Sjdp#ifndef FIRE_DEBUG 21733965Sjdp#define FIRE_SPC_SET(spc, sc, offs, reg, v) \ 21833965Sjdp FIRE_SPC_WRITE_8((spc), (sc), (offs), (v)) 21933965Sjdp#else 22033965Sjdp#define FIRE_SPC_SET(spc, sc, offs, reg, v) do { \ 22133965Sjdp device_printf((sc)->sc_dev, reg " 0x%016llx -> 0x%016llx\n", \ 22233965Sjdp (unsigned long long)FIRE_SPC_READ_8((spc), (sc), (offs)), \ 22333965Sjdp (unsigned long long)(v)); \ 22433965Sjdp FIRE_SPC_WRITE_8((spc), (sc), (offs), (v)); \ 22533965Sjdp } while (0) 226130561Sobrien#endif 22733965Sjdp 22833965Sjdp#define FIRE_PCI_BARRIER(sc, offs, len, flags) \ 22933965Sjdp FIRE_SPC_BARRIER(FIRE_PCI, (sc), (offs), len, flags) 23033965Sjdp#define FIRE_PCI_READ_8(sc, offs) \ 23133965Sjdp FIRE_SPC_READ_8(FIRE_PCI, (sc), (offs)) 232130561Sobrien#define FIRE_PCI_WRITE_8(sc, offs, v) \ 23333965Sjdp FIRE_SPC_WRITE_8(FIRE_PCI, (sc), (offs), (v)) 23433965Sjdp#define FIRE_CTRL_BARRIER(sc, offs, len, flags) \ 23533965Sjdp FIRE_SPC_BARRIER(FIRE_CTRL, (sc), (offs), len, flags) 23633965Sjdp#define FIRE_CTRL_READ_8(sc, offs) \ 23733965Sjdp FIRE_SPC_READ_8(FIRE_CTRL, (sc), (offs)) 23833965Sjdp#define FIRE_CTRL_WRITE_8(sc, offs, v) \ 23960484Sobrien FIRE_SPC_WRITE_8(FIRE_CTRL, (sc), (offs), (v)) 24060484Sobrien 24160484Sobrien#define FIRE_PCI_SET(sc, offs, v) \ 24260484Sobrien FIRE_SPC_SET(FIRE_PCI, (sc), (offs), # offs, (v)) 24360484Sobrien#define FIRE_CTRL_SET(sc, offs, v) \ 24460484Sobrien FIRE_SPC_SET(FIRE_CTRL, (sc), (offs), # offs, (v)) 24560484Sobrien 24660484Sobrienstruct fire_desc { 24733965Sjdp const char *fd_string; 24833965Sjdp int fd_mode; 24933965Sjdp const char *fd_name; 25033965Sjdp}; 25133965Sjdp 25233965Sjdpstatic const struct fire_desc fire_compats[] = { 253130561Sobrien { "pciex108e,80f0", FIRE_MODE_FIRE, "Fire" }, 25433965Sjdp#if 0 25533965Sjdp { "pciex108e,80f8", FIRE_MODE_OBERON, "Oberon" }, 256130561Sobrien#endif 25733965Sjdp { NULL, 0, NULL } 25833965Sjdp}; 25933965Sjdp 26033965Sjdpstatic const struct fire_desc * 26133965Sjdpfire_get_desc(device_t dev) 262130561Sobrien{ 26333965Sjdp const struct fire_desc *desc; 26433965Sjdp const char *compat; 26533965Sjdp 26633965Sjdp compat = ofw_bus_get_compat(dev); 267130561Sobrien if (compat == NULL) 26833965Sjdp return (NULL); 26933965Sjdp for (desc = fire_compats; desc->fd_string != NULL; desc++) 27033965Sjdp if (strcmp(desc->fd_string, compat) == 0) 27133965Sjdp return (desc); 27233965Sjdp return (NULL); 27333965Sjdp} 27433965Sjdp 27533965Sjdpstatic int 27633965Sjdpfire_probe(device_t dev) 27733965Sjdp{ 27833965Sjdp const char *dtype; 27933965Sjdp 28033965Sjdp dtype = ofw_bus_get_type(dev); 28133965Sjdp if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCIE) == 0 && 28233965Sjdp fire_get_desc(dev) != NULL) { 28333965Sjdp device_set_desc(dev, "Sun Host-PCIe bridge"); 28433965Sjdp return (BUS_PROBE_GENERIC); 28533965Sjdp } 28633965Sjdp return (ENXIO); 28733965Sjdp} 28833965Sjdp 289130561Sobrienstatic int 29033965Sjdpfire_attach(device_t dev) 29133965Sjdp{ 29233965Sjdp struct fire_softc *sc; 29333965Sjdp const struct fire_desc *desc; 29433965Sjdp struct ofw_pci_msi_ranges msi_ranges; 29533965Sjdp struct ofw_pci_msi_addr_ranges msi_addr_ranges; 29689857Sobrien struct ofw_pci_msi_eq_to_devino msi_eq_to_devino; 29733965Sjdp struct fire_msiqarg *fmqa; 29889857Sobrien struct timecounter *tc; 29933965Sjdp struct ofw_pci_ranges *range; 30089857Sobrien uint64_t ino_bitmap, val; 30133965Sjdp phandle_t node; 30233965Sjdp uint32_t prop, prop_array[2]; 30333965Sjdp int i, j, mode; 30433965Sjdp u_int lw; 30533965Sjdp uint16_t mps; 30633965Sjdp 30733965Sjdp sc = device_get_softc(dev); 30833965Sjdp node = ofw_bus_get_node(dev); 30933965Sjdp desc = fire_get_desc(dev); 310130561Sobrien mode = desc->fd_mode; 31133965Sjdp 31233965Sjdp sc->sc_dev = dev; 31333965Sjdp sc->sc_node = node; 31433965Sjdp sc->sc_mode = mode; 31533965Sjdp sc->sc_flags = 0; 31633965Sjdp 31733965Sjdp mtx_init(&sc->sc_msi_mtx, "msi_mtx", NULL, MTX_DEF); 31833965Sjdp mtx_init(&sc->sc_pcib_mtx, "pcib_mtx", NULL, MTX_SPIN); 31933965Sjdp 32033965Sjdp /* 32133965Sjdp * Fire and Oberon have two register banks: 32233965Sjdp * (0) per-PBM PCI Express configuration and status registers 32333965Sjdp * (1) (shared) Fire/Oberon controller configuration and status 32433965Sjdp * registers 32533965Sjdp */ 326130561Sobrien for (i = 0; i < FIRE_NREG; i++) { 32733965Sjdp j = i; 32860484Sobrien sc->sc_mem_res[i] = bus_alloc_resource_any(dev, 32933965Sjdp SYS_RES_MEMORY, &j, RF_ACTIVE); 33033965Sjdp if (sc->sc_mem_res[i] == NULL) 33133965Sjdp panic("%s: could not allocate register bank %d", 33233965Sjdp __func__, i); 33333965Sjdp } 33433965Sjdp 33533965Sjdp if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1) 336130561Sobrien panic("%s: could not determine IGN", __func__); 33733965Sjdp if (OF_getprop(node, "module-revision#", &prop, sizeof(prop)) == -1) 33860484Sobrien panic("%s: could not determine module-revision", __func__); 33933965Sjdp 34033965Sjdp device_printf(dev, "%s, module-revision %d, IGN %#x\n", 34133965Sjdp desc->fd_name, prop, sc->sc_ign); 34233965Sjdp 34333965Sjdp /* 344130561Sobrien * Hunt through all the interrupt mapping regs and register 34533965Sjdp * the interrupt controller for our interrupt vectors. We do 34660484Sobrien * this early in order to be able to catch stray interrupts. 34733965Sjdp */ 34833965Sjdp i = OF_getprop(node, "ino-bitmap", (void *)prop_array, 34933965Sjdp sizeof(prop_array)); 35033965Sjdp if (i == -1) 351130561Sobrien panic("%s: could not get ino-bitmap", __func__); 352130561Sobrien ino_bitmap = ((uint64_t)prop_array[1] << 32) | prop_array[0]; 353130561Sobrien for (i = 0; i <= FO_MAX_INO; i++) { 35433965Sjdp if ((ino_bitmap & (1ULL << i)) == 0) 35533965Sjdp continue; 35633965Sjdp j = fire_intr_register(sc, i); 35733965Sjdp if (j != 0) 35833965Sjdp device_printf(dev, "could not register interrupt " 35933965Sjdp "controller for INO %d (%d)\n", i, j); 36033965Sjdp } 36133965Sjdp 36233965Sjdp /* JBC/UBC module initialization */ 36333965Sjdp FIRE_CTRL_SET(sc, FO_XBC_ERR_LOG_EN, ~0ULL); 36433965Sjdp FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); 36533965Sjdp /* not enabled by OpenSolaris */ 36633965Sjdp FIRE_CTRL_SET(sc, FO_XBC_INT_EN, ~0ULL); 367130561Sobrien if (sc->sc_mode == FIRE_MODE_FIRE) { 36833965Sjdp FIRE_CTRL_SET(sc, FIRE_JBUS_PAR_CTRL, 36933965Sjdp FIRE_JBUS_PAR_CTRL_P_EN); 37033965Sjdp FIRE_CTRL_SET(sc, FIRE_JBC_FATAL_RST_EN, 37133965Sjdp ((1ULL << FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_SHFT) & 37233965Sjdp FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_MASK) | 373130561Sobrien FIRE_JBC_FATAL_RST_EN_MB_PEA_P_INT | 374130561Sobrien FIRE_JBC_FATAL_RST_EN_CPE_P_INT | 37533965Sjdp FIRE_JBC_FATAL_RST_EN_APE_P_INT | 37633965Sjdp FIRE_JBC_FATAL_RST_EN_PIO_CPE_INT | 37733965Sjdp FIRE_JBC_FATAL_RST_EN_JTCEEW_P_INT | 37833965Sjdp FIRE_JBC_FATAL_RST_EN_JTCEEI_P_INT | 37933965Sjdp FIRE_JBC_FATAL_RST_EN_JTCEER_P_INT); 38033965Sjdp FIRE_CTRL_SET(sc, FIRE_JBC_CORE_BLOCK_INT_EN, ~0ULL); 38133965Sjdp } 38233965Sjdp 383130561Sobrien /* TLU initialization */ 384130561Sobrien FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_STAT_CLR, 38533965Sjdp FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK); 38633965Sjdp /* not enabled by OpenSolaris */ 38733965Sjdp FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_INT_EN, 38833965Sjdp FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK); 38933965Sjdp FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_STAT_CLR, 39033965Sjdp FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK); 39133965Sjdp /* not enabled by OpenSolaris */ 39233965Sjdp FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_INT_EN, 39333965Sjdp FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK); 39433965Sjdp FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_STAT_CLR, 39533965Sjdp FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK); 39633965Sjdp /* not enabled by OpenSolaris */ 397130561Sobrien FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_INT_EN, 39833965Sjdp FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK); 39933965Sjdp val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) | 400130561Sobrien ((FO_PCI_TLU_CTRL_L0S_TIM_DFLT << FO_PCI_TLU_CTRL_L0S_TIM_SHFT) & 40133965Sjdp FO_PCI_TLU_CTRL_L0S_TIM_MASK) | 40233965Sjdp ((FO_PCI_TLU_CTRL_CFG_DFLT << FO_PCI_TLU_CTRL_CFG_SHFT) & 40333965Sjdp FO_PCI_TLU_CTRL_CFG_MASK); 40433965Sjdp if (sc->sc_mode == FIRE_MODE_OBERON) 405130561Sobrien val &= ~FO_PCI_TLU_CTRL_NWPR_EN; 406130561Sobrien val |= FO_PCI_TLU_CTRL_CFG_REMAIN_DETECT_QUIET; 407130561Sobrien FIRE_PCI_SET(sc, FO_PCI_TLU_CTRL, val); 40833965Sjdp FIRE_PCI_SET(sc, FO_PCI_TLU_DEV_CTRL, 0); 40933965Sjdp FIRE_PCI_SET(sc, FO_PCI_TLU_LNK_CTRL, FO_PCI_TLU_LNK_CTRL_CLK); 41033965Sjdp 41133965Sjdp /* DLU/LPU initialization */ 41233965Sjdp if (sc->sc_mode == FIRE_MODE_OBERON) 41333965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_INT_MASK, 0); 41433965Sjdp else 41533965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_RST, 0); 41633965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_LNK_LYR_CFG, 41733965Sjdp FO_PCI_LPU_LNK_LYR_CFG_VC0_EN); 41833965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_FLW_CTRL_UPDT_CTRL, 419130561Sobrien FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_NP_EN | 42033965Sjdp FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_P_EN); 42133965Sjdp if (sc->sc_mode == FIRE_MODE_OBERON) 42233965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS, 423130561Sobrien (OBERON_PCI_LPU_TXLNK_RPLY_TMR_THRS_DFLT << 42433965Sjdp FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) & 42533965Sjdp FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK); 42633965Sjdp else { 42733965Sjdp switch ((FIRE_PCI_READ_8(sc, FO_PCI_TLU_LNK_STAT) & 42833965Sjdp FO_PCI_TLU_LNK_STAT_WDTH_MASK) >> 42933965Sjdp FO_PCI_TLU_LNK_STAT_WDTH_SHFT) { 43033965Sjdp case 1: 43133965Sjdp lw = 0; 43233965Sjdp break; 43333965Sjdp case 4: 43433965Sjdp lw = 1; 43533965Sjdp break; 43633965Sjdp case 8: 43733965Sjdp lw = 2; 43833965Sjdp break; 43933965Sjdp case 16: 44033965Sjdp lw = 3; 44133965Sjdp break; 44233965Sjdp default: 44333965Sjdp lw = 0; 44433965Sjdp } 44533965Sjdp mps = (FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) & 44633965Sjdp FO_PCI_TLU_CTRL_CFG_MPS_MASK) >> 44733965Sjdp FO_PCI_TLU_CTRL_CFG_MPS_SHFT; 44833965Sjdp i = sizeof(fire_freq_nak_tmr_thrs) / 44933965Sjdp sizeof(*fire_freq_nak_tmr_thrs); 45033965Sjdp if (mps >= i) 45133965Sjdp mps = i - 1; 45233965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS, 45333965Sjdp (fire_freq_nak_tmr_thrs[mps][lw] << 45433965Sjdp FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_SHFT) & 45533965Sjdp FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_MASK); 45660484Sobrien FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS, 457130561Sobrien (fire_rply_tmr_thrs[mps][lw] << 45833965Sjdp FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) & 45933965Sjdp FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK); 46033965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RTR_FIFO_PTR, 46133965Sjdp ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_DFLT << 46233965Sjdp FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_SHFT) & 46333965Sjdp FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_MASK) | 46433965Sjdp ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_DFLT << 465130561Sobrien FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_SHFT) & 46633965Sjdp FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_MASK)); 46733965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG2, 46833965Sjdp (FO_PCI_LPU_LTSSM_CFG2_12_TO_DFLT << 469130561Sobrien FO_PCI_LPU_LTSSM_CFG2_12_TO_SHFT) & 47033965Sjdp FO_PCI_LPU_LTSSM_CFG2_12_TO_MASK); 47133965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG3, 47233965Sjdp (FO_PCI_LPU_LTSSM_CFG3_2_TO_DFLT << 47333965Sjdp FO_PCI_LPU_LTSSM_CFG3_2_TO_SHFT) & 47433965Sjdp FO_PCI_LPU_LTSSM_CFG3_2_TO_MASK); 47533965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG4, 47633965Sjdp ((FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_DFLT << 47733965Sjdp FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_SHFT) & 47833965Sjdp FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_MASK) | 47933965Sjdp ((FO_PCI_LPU_LTSSM_CFG4_N_FTS_DFLT << 48033965Sjdp FO_PCI_LPU_LTSSM_CFG4_N_FTS_SHFT) & 48133965Sjdp FO_PCI_LPU_LTSSM_CFG4_N_FTS_MASK)); 48233965Sjdp FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG5, 0); 483130561Sobrien } 48433965Sjdp 48533965Sjdp /* ILU initialization */ 48633965Sjdp FIRE_PCI_SET(sc, FO_PCI_ILU_ERR_STAT_CLR, ~0ULL); 48733965Sjdp /* not enabled by OpenSolaris */ 48833965Sjdp FIRE_PCI_SET(sc, FO_PCI_ILU_INT_EN, ~0ULL); 48933965Sjdp 490130561Sobrien /* IMU initialization */ 49133965Sjdp FIRE_PCI_SET(sc, FO_PCI_IMU_ERR_STAT_CLR, ~0ULL); 49233965Sjdp FIRE_PCI_SET(sc, FO_PCI_IMU_INT_EN, 49333965Sjdp FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_EN) & 49433965Sjdp ~(FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_S | 49560484Sobrien FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_S | 496130561Sobrien FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_S | 49733965Sjdp FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P | 49833965Sjdp FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P | 49933965Sjdp FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P)); 50033965Sjdp 50133965Sjdp /* MMU initialization */ 50233965Sjdp FIRE_PCI_SET(sc, FO_PCI_MMU_ERR_STAT_CLR, 50333965Sjdp FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK); 50433965Sjdp /* not enabled by OpenSolaris */ 50533965Sjdp FIRE_PCI_SET(sc, FO_PCI_MMU_INT_EN, 50633965Sjdp FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK); 50733965Sjdp 50833965Sjdp /* DMC initialization */ 50933965Sjdp FIRE_PCI_SET(sc, FO_PCI_DMC_CORE_BLOCK_INT_EN, ~0ULL); 51033965Sjdp FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTA, 0); 51133965Sjdp FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTB, 0); 51233965Sjdp 513130561Sobrien /* PEC initialization */ 514130561Sobrien FIRE_PCI_SET(sc, FO_PCI_PEC_CORE_BLOCK_INT_EN, ~0ULL); 51533965Sjdp 51633965Sjdp /* Establish handlers for interesting interrupts. */ 51733965Sjdp if ((ino_bitmap & (1ULL << FO_DMC_PEC_INO)) != 0) 51833965Sjdp fire_set_intr(sc, 1, FO_DMC_PEC_INO, fire_dmc_pec, sc); 51933965Sjdp if ((ino_bitmap & (1ULL << FO_XCB_INO)) != 0) 52033965Sjdp fire_set_intr(sc, 0, FO_XCB_INO, fire_xcb, sc); 521130561Sobrien 52233965Sjdp /* MSI/MSI-X support */ 52333965Sjdp if (OF_getprop(node, "#msi", &sc->sc_msi_count, 52460484Sobrien sizeof(sc->sc_msi_count)) == -1) 52560484Sobrien panic("%s: could not determine MSI count", __func__); 52633965Sjdp if (OF_getprop(node, "msi-ranges", &msi_ranges, 52733965Sjdp sizeof(msi_ranges)) == -1) 52833965Sjdp sc->sc_msi_first = 0; 52933965Sjdp else 53033965Sjdp sc->sc_msi_first = msi_ranges.first; 53133965Sjdp if (OF_getprop(node, "msi-data-mask", &sc->sc_msi_data_mask, 53233965Sjdp sizeof(sc->sc_msi_data_mask)) == -1) 53333965Sjdp panic("%s: could not determine MSI data mask", __func__); 53460484Sobrien if (OF_getprop(node, "msix-data-width", &sc->sc_msix_data_width, 535104834Sobrien sizeof(sc->sc_msix_data_width)) > 0) 53633965Sjdp sc->sc_flags |= FIRE_MSIX; 53733965Sjdp if (OF_getprop(node, "msi-address-ranges", &msi_addr_ranges, 53833965Sjdp sizeof(msi_addr_ranges)) == -1) 53933965Sjdp panic("%s: could not determine MSI address ranges", __func__); 54033965Sjdp sc->sc_msi_addr32 = OFW_PCI_MSI_ADDR_RANGE_32(&msi_addr_ranges); 54133965Sjdp sc->sc_msi_addr64 = OFW_PCI_MSI_ADDR_RANGE_64(&msi_addr_ranges); 54233965Sjdp if (OF_getprop(node, "#msi-eqs", &sc->sc_msiq_count, 54333965Sjdp sizeof(sc->sc_msiq_count)) == -1) 54433965Sjdp panic("%s: could not determine MSI event queue count", 54533965Sjdp __func__); 54633965Sjdp if (OF_getprop(node, "msi-eq-size", &sc->sc_msiq_size, 54733965Sjdp sizeof(sc->sc_msiq_size)) == -1) 54833965Sjdp panic("%s: could not determine MSI event queue size", 54933965Sjdp __func__); 550130561Sobrien if (OF_getprop(node, "msi-eq-to-devino", &msi_eq_to_devino, 55133965Sjdp sizeof(msi_eq_to_devino)) == -1 && 55233965Sjdp OF_getprop(node, "msi-eq-devino", &msi_eq_to_devino, 55333965Sjdp sizeof(msi_eq_to_devino)) == -1) { 55433965Sjdp sc->sc_msiq_first = 0; 55533965Sjdp sc->sc_msiq_ino_first = FO_EQ_FIRST_INO; 55633965Sjdp } else { 557130561Sobrien sc->sc_msiq_first = msi_eq_to_devino.eq_first; 55833965Sjdp sc->sc_msiq_ino_first = msi_eq_to_devino.devino_first; 55933965Sjdp } 56033965Sjdp if (sc->sc_msiq_ino_first < FO_EQ_FIRST_INO || 56133965Sjdp sc->sc_msiq_ino_first + sc->sc_msiq_count - 1 > FO_EQ_LAST_INO) 56233965Sjdp panic("%s: event queues exceed INO range", __func__); 563130561Sobrien sc->sc_msi_bitmap = malloc(roundup2(sc->sc_msi_count, NBBY) / NBBY, 56433965Sjdp M_DEVBUF, M_NOWAIT | M_ZERO); 56533965Sjdp if (sc->sc_msi_bitmap == NULL) 56633965Sjdp panic("%s: could not malloc MSI bitmap", __func__); 56733965Sjdp sc->sc_msi_msiq_table = malloc(sc->sc_msi_count * 56833965Sjdp sizeof(*sc->sc_msi_msiq_table), M_DEVBUF, M_NOWAIT | M_ZERO); 56933965Sjdp if (sc->sc_msi_msiq_table == NULL) 570130561Sobrien panic("%s: could not malloc MSI-MSI event queue table", 57133965Sjdp __func__); 57233965Sjdp sc->sc_msiq_bitmap = malloc(roundup2(sc->sc_msiq_count, NBBY) / NBBY, 57333965Sjdp M_DEVBUF, M_NOWAIT | M_ZERO); 57433965Sjdp if (sc->sc_msiq_bitmap == NULL) 575130561Sobrien panic("%s: could not malloc MSI event queue bitmap", __func__); 576130561Sobrien j = FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * sc->sc_msiq_count; 577130561Sobrien sc->sc_msiq = contigmalloc(j, M_DEVBUF, M_NOWAIT, 0, ~0UL, 57833965Sjdp FO_EQ_ALIGNMENT, 0); 57933965Sjdp if (sc->sc_msiq == NULL) 58033965Sjdp panic("%s: could not contigmalloc MSI event queue", __func__); 58133965Sjdp memset(sc->sc_msiq, 0, j); 582130561Sobrien FIRE_PCI_SET(sc, FO_PCI_EQ_BASE_ADDR, FO_PCI_EQ_BASE_ADDR_BYPASS | 58333965Sjdp (pmap_kextract((vm_offset_t)sc->sc_msiq) & 58433965Sjdp FO_PCI_EQ_BASE_ADDR_MASK)); 58533965Sjdp for (i = 0; i < sc->sc_msi_count; i++) { 58633965Sjdp j = (i + sc->sc_msi_first) << 3; 587130561Sobrien FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + j, 58833965Sjdp FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + j) & 58933965Sjdp ~FO_PCI_MSI_MAP_V); 59033965Sjdp } 59133965Sjdp for (i = 0; i < sc->sc_msiq_count; i++) { 59233965Sjdp j = i + sc->sc_msiq_ino_first; 59333965Sjdp if ((ino_bitmap & (1ULL << j)) == 0) { 59433965Sjdp mtx_lock(&sc->sc_msi_mtx); 59533965Sjdp setbit(sc->sc_msiq_bitmap, i); 59633965Sjdp mtx_unlock(&sc->sc_msi_mtx); 59733965Sjdp } 59833965Sjdp fmqa = intr_vectors[INTMAP_VEC(sc->sc_ign, j)].iv_icarg; 59933965Sjdp mtx_init(&fmqa->fmqa_mtx, "msiq_mtx", NULL, MTX_SPIN); 60033965Sjdp fmqa->fmqa_base = 601130561Sobrien (struct fo_msiq_record *)((caddr_t)sc->sc_msiq + 602130561Sobrien (FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * i)); 60333965Sjdp j = i + sc->sc_msiq_first; 60433965Sjdp fmqa->fmqa_msiq = j; 60533965Sjdp j <<= 3; 60633965Sjdp fmqa->fmqa_head = FO_PCI_EQ_HD_BASE + j; 60733965Sjdp fmqa->fmqa_tail = FO_PCI_EQ_TL_BASE + j; 60833965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + j, 60933965Sjdp FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I | 61033965Sjdp FO_PCI_EQ_CTRL_CLR_DIS); 61133965Sjdp FIRE_PCI_WRITE_8(sc, fmqa->fmqa_tail, 61233965Sjdp (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK); 61333965Sjdp FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, 61433965Sjdp (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK); 61533965Sjdp } 61633965Sjdp FIRE_PCI_SET(sc, FO_PCI_MSI_32_BIT_ADDR, sc->sc_msi_addr32 & 61733965Sjdp FO_PCI_MSI_32_BIT_ADDR_MASK); 61833965Sjdp FIRE_PCI_SET(sc, FO_PCI_MSI_64_BIT_ADDR, sc->sc_msi_addr64 & 61933965Sjdp FO_PCI_MSI_64_BIT_ADDR_MASK); 62033965Sjdp 62133965Sjdp /* 62233965Sjdp * Establish a handler for interesting PCIe messages and disable 62333965Sjdp * unintersting ones. 62433965Sjdp */ 62533965Sjdp mtx_lock(&sc->sc_msi_mtx); 62633965Sjdp for (i = 0; i < sc->sc_msiq_count; i++) { 62733965Sjdp if (isclr(sc->sc_msiq_bitmap, i) != 0) { 62833965Sjdp j = i; 62933965Sjdp break; 63033965Sjdp } 63133965Sjdp } 63233965Sjdp if (i == sc->sc_msiq_count) { 63333965Sjdp mtx_unlock(&sc->sc_msi_mtx); 63433965Sjdp panic("%s: no spare event queue for PCIe messages", __func__); 63533965Sjdp } 63633965Sjdp setbit(sc->sc_msiq_bitmap, j); 63733965Sjdp mtx_unlock(&sc->sc_msi_mtx); 63833965Sjdp i = INTMAP_VEC(sc->sc_ign, j + sc->sc_msiq_ino_first); 63933965Sjdp if (bus_set_resource(dev, SYS_RES_IRQ, 2, i, 1) != 0) 64033965Sjdp panic("%s: failed to add interrupt for PCIe messages", 64133965Sjdp __func__); 642130561Sobrien fire_set_intr(sc, 2, INTINO(i), fire_pcie, intr_vectors[i].iv_icarg); 64333965Sjdp j += sc->sc_msiq_first; 64433965Sjdp /* 64533965Sjdp * "Please note that setting the EQNUM field to a value larger than 64633965Sjdp * 35 will yield unpredictable results." 64733965Sjdp */ 64833965Sjdp if (j > 35) 649130561Sobrien panic("%s: invalid queue for PCIe messages (%d)", 65033965Sjdp __func__, j); 65133965Sjdp FIRE_PCI_SET(sc, FO_PCI_ERR_COR, FO_PCI_ERR_PME_V | 65233965Sjdp ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); 653130561Sobrien FIRE_PCI_SET(sc, FO_PCI_ERR_NONFATAL, FO_PCI_ERR_PME_V | 65433965Sjdp ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); 65533965Sjdp FIRE_PCI_SET(sc, FO_PCI_ERR_FATAL, FO_PCI_ERR_PME_V | 65633965Sjdp ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); 65733965Sjdp FIRE_PCI_SET(sc, FO_PCI_PM_PME, 0); 65833965Sjdp FIRE_PCI_SET(sc, FO_PCI_PME_TO_ACK, 0); 65933965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (j << 3), 66033965Sjdp FO_PCI_EQ_CTRL_SET_EN); 66133965Sjdp 66233965Sjdp#define TC_COUNTER_MAX_MASK 0xffffffff 663130561Sobrien 66433965Sjdp /* 66533965Sjdp * Setup JBC/UBC performance counter 0 in bus cycle counting 66633965Sjdp * mode as timecounter. 66733965Sjdp */ 66833965Sjdp if (device_get_unit(dev) == 0) { 66977298Sobrien FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT0, 0); 67033965Sjdp FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT1, 0); 67133965Sjdp FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT_SEL, 67233965Sjdp (FO_XBC_PRF_CNT_NONE << FO_XBC_PRF_CNT_CNT1_SHFT) | 673130561Sobrien (FO_XBC_PRF_CNT_XB_CLK << FO_XBC_PRF_CNT_CNT0_SHFT)); 67433965Sjdp tc = malloc(sizeof(*tc), M_DEVBUF, M_NOWAIT | M_ZERO); 67533965Sjdp if (tc == NULL) 67633965Sjdp panic("%s: could not malloc timecounter", __func__); 67733965Sjdp tc->tc_get_timecount = fire_get_timecount; 678130561Sobrien tc->tc_counter_mask = TC_COUNTER_MAX_MASK; 679130561Sobrien if (OF_getprop(OF_peer(0), "clock-frequency", &prop, 680130561Sobrien sizeof(prop)) == -1) 68133965Sjdp panic("%s: could not determine clock frequency", 68233965Sjdp __func__); 68333965Sjdp tc->tc_frequency = prop; 68433965Sjdp tc->tc_name = strdup(device_get_nameunit(dev), M_DEVBUF); 68533965Sjdp tc->tc_priv = sc; 686130561Sobrien /* 687130561Sobrien * Due to initial problems with the JBus-driven performance 68833965Sjdp * counters not advancing which might be firmware dependent 68933965Sjdp * ensure that it actually works. 69033965Sjdp */ 69133965Sjdp if (fire_get_timecount(tc) - fire_get_timecount(tc) != 0) 69233965Sjdp tc->tc_quality = FIRE_PERF_CNT_QLTY; 693130561Sobrien else 69433965Sjdp tc->tc_quality = -FIRE_PERF_CNT_QLTY; 69533965Sjdp tc_init(tc); 69633965Sjdp } 69733965Sjdp 69833965Sjdp /* 69933965Sjdp * Set up the IOMMU. Both Fire and Oberon have one per PBM, but 70033965Sjdp * neither has a streaming buffer. 70133965Sjdp */ 702130561Sobrien memcpy(&sc->sc_dma_methods, &iommu_dma_methods, 70333965Sjdp sizeof(sc->sc_dma_methods)); 70433965Sjdp sc->sc_is.is_flags = IOMMU_FIRE | IOMMU_PRESERVE_PROM; 70533965Sjdp if (sc->sc_mode == FIRE_MODE_OBERON) { 70633965Sjdp sc->sc_is.is_flags |= IOMMU_FLUSH_CACHE; 70733965Sjdp sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(OBERON_IOMMU_BITS); 70833965Sjdp } else { 70933965Sjdp sc->sc_dma_methods.dm_dmamap_sync = fire_dmamap_sync; 71033965Sjdp sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(FIRE_IOMMU_BITS); 71133965Sjdp } 71233965Sjdp sc->sc_is.is_sb[0] = sc->sc_is.is_sb[1] = 0; 71333965Sjdp /* Punch in our copies. */ 71433965Sjdp sc->sc_is.is_bustag = rman_get_bustag(sc->sc_mem_res[FIRE_PCI]); 71533965Sjdp sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[FIRE_PCI]); 71633965Sjdp sc->sc_is.is_iommu = FO_PCI_MMU; 71733965Sjdp val = FIRE_PCI_READ_8(sc, FO_PCI_MMU + IMR_CTL); 71833965Sjdp iommu_init(device_get_nameunit(sc->sc_dev), &sc->sc_is, 7, -1, 0); 71933965Sjdp#ifdef FIRE_DEBUG 72033965Sjdp device_printf(dev, "FO_PCI_MMU + IMR_CTL 0x%016llx -> 0x%016llx\n", 72133965Sjdp (long long unsigned)val, (long long unsigned)sc->sc_is.is_cr); 72233965Sjdp#endif 72333965Sjdp 72433965Sjdp /* Initialize memory and I/O rmans. */ 72533965Sjdp sc->sc_pci_io_rman.rm_type = RMAN_ARRAY; 72633965Sjdp sc->sc_pci_io_rman.rm_descr = "Fire PCI I/O Ports"; 72733965Sjdp if (rman_init(&sc->sc_pci_io_rman) != 0 || 72833965Sjdp rman_manage_region(&sc->sc_pci_io_rman, 0, FO_IO_SIZE) != 0) 72933965Sjdp panic("%s: failed to set up I/O rman", __func__); 73033965Sjdp sc->sc_pci_mem_rman.rm_type = RMAN_ARRAY; 73133965Sjdp sc->sc_pci_mem_rman.rm_descr = "Fire PCI Memory"; 73233965Sjdp if (rman_init(&sc->sc_pci_mem_rman) != 0 || 73333965Sjdp rman_manage_region(&sc->sc_pci_mem_rman, 0, FO_MEM_SIZE) != 0) 73433965Sjdp panic("%s: failed to set up memory rman", __func__); 73533965Sjdp 73660484Sobrien i = OF_getprop_alloc(node, "ranges", sizeof(*range), (void **)&range); 73733965Sjdp /* 73833965Sjdp * Make sure that the expected ranges are present. The 73933965Sjdp * OFW_PCI_CS_MEM64 one is not currently used though. 74033965Sjdp */ 74133965Sjdp if (i != FIRE_NRANGE) 74233965Sjdp panic("%s: unsupported number of ranges", __func__); 74333965Sjdp /* 74433965Sjdp * Find the addresses of the various bus spaces. 74533965Sjdp * There should not be multiple ones of one kind. 74633965Sjdp * The physical start addresses of the ranges are the configuration, 74733965Sjdp * memory and I/O handles. 74833965Sjdp */ 74933965Sjdp for (i = 0; i < FIRE_NRANGE; i++) { 75089857Sobrien j = OFW_PCI_RANGE_CS(&range[i]); 75133965Sjdp if (sc->sc_pci_bh[j] != 0) 75233965Sjdp panic("%s: duplicate range for space %d", 75333965Sjdp __func__, j); 75433965Sjdp sc->sc_pci_bh[j] = OFW_PCI_RANGE_PHYS(&range[i]); 75533965Sjdp } 75633965Sjdp free(range, M_OFWPROP); 75733965Sjdp 75833965Sjdp /* Allocate our tags. */ 75933965Sjdp sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, rman_get_bustag( 76033965Sjdp sc->sc_mem_res[FIRE_PCI]), PCI_IO_BUS_SPACE, NULL); 76133965Sjdp if (sc->sc_pci_iot == NULL) 76233965Sjdp panic("%s: could not allocate PCI I/O tag", __func__); 76333965Sjdp sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, rman_get_bustag( 76433965Sjdp sc->sc_mem_res[FIRE_PCI]), PCI_CONFIG_BUS_SPACE, NULL); 76533965Sjdp if (sc->sc_pci_cfgt == NULL) 76633965Sjdp panic("%s: could not allocate PCI configuration space tag", 767130561Sobrien __func__); 76833965Sjdp if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0x100000000, 76933965Sjdp sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr, 77033965Sjdp 0xff, 0xffffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0) 77133965Sjdp panic("%s: could not create PCI DMA tag", __func__); 77233965Sjdp /* Customize the tag. */ 77333965Sjdp sc->sc_pci_dmat->dt_cookie = &sc->sc_is; 77433965Sjdp sc->sc_pci_dmat->dt_mt = &sc->sc_dma_methods; 775130561Sobrien 77633965Sjdp /* 77733965Sjdp * Get the bus range from the firmware. 77833965Sjdp * NB: Neither Fire nor Oberon support PCI bus reenumeration. 77933965Sjdp */ 78033965Sjdp i = OF_getprop(node, "bus-range", (void *)prop_array, 78133965Sjdp sizeof(prop_array)); 78233965Sjdp if (i == -1) 78333965Sjdp panic("%s: could not get bus-range", __func__); 78433965Sjdp if (i != sizeof(prop_array)) 78533965Sjdp panic("%s: broken bus-range (%d)", __func__, i); 786130561Sobrien sc->sc_pci_secbus = prop_array[0]; 78733965Sjdp sc->sc_pci_subbus = prop_array[1]; 78833965Sjdp if (bootverbose != 0) 78933965Sjdp device_printf(dev, "bus range %u to %u; PCI bus %d\n", 79033965Sjdp sc->sc_pci_secbus, sc->sc_pci_subbus, sc->sc_pci_secbus); 79133965Sjdp 79233965Sjdp ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(ofw_pci_intr_t)); 79333965Sjdp 79433965Sjdp#define FIRE_SYSCTL_ADD_UINT(name, arg, desc) \ 79533965Sjdp SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), \ 796130561Sobrien SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, \ 79733965Sjdp (name), CTLFLAG_RD, (arg), 0, (desc)) 79833965Sjdp 79933965Sjdp FIRE_SYSCTL_ADD_UINT("ilu_err", &sc->sc_stats_ilu_err, 800130561Sobrien "ILU unknown errors"); 80133965Sjdp FIRE_SYSCTL_ADD_UINT("jbc_ce_async", &sc->sc_stats_jbc_ce_async, 80233965Sjdp "JBC correctable errors"); 803130561Sobrien FIRE_SYSCTL_ADD_UINT("jbc_unsol_int", &sc->sc_stats_jbc_unsol_int, 80433965Sjdp "JBC unsolicited interrupt ACK/NACK errors"); 80533965Sjdp FIRE_SYSCTL_ADD_UINT("jbc_unsol_rd", &sc->sc_stats_jbc_unsol_rd, 80633965Sjdp "JBC unsolicited read response errors"); 807130561Sobrien FIRE_SYSCTL_ADD_UINT("mmu_err", &sc->sc_stats_mmu_err, "MMU errors"); 80833965Sjdp FIRE_SYSCTL_ADD_UINT("tlu_ce", &sc->sc_stats_tlu_ce, 80933965Sjdp "DLU/TLU correctable errors"); 81033965Sjdp FIRE_SYSCTL_ADD_UINT("tlu_oe_non_fatal", 81133965Sjdp &sc->sc_stats_tlu_oe_non_fatal, 81233965Sjdp "DLU/TLU other event non-fatal errors summary"), 81333965Sjdp FIRE_SYSCTL_ADD_UINT("tlu_oe_rx_err", &sc->sc_stats_tlu_oe_rx_err, 81433965Sjdp "DLU/TLU receive other event errors"), 81533965Sjdp FIRE_SYSCTL_ADD_UINT("tlu_oe_tx_err", &sc->sc_stats_tlu_oe_tx_err, 81633965Sjdp "DLU/TLU transmit other event errors"), 817130561Sobrien FIRE_SYSCTL_ADD_UINT("ubc_dmardue", &sc->sc_stats_ubc_dmardue, 81833965Sjdp "UBC DMARDUE erros"); 819130561Sobrien 82033965Sjdp#undef FIRE_SYSCTL_ADD_UINT 82133965Sjdp 82233965Sjdp device_add_child(dev, "pci", -1); 82333965Sjdp return (bus_generic_attach(dev)); 82433965Sjdp} 82533965Sjdp 82633965Sjdpstatic void 82733965Sjdpfire_set_intr(struct fire_softc *sc, u_int index, u_int ino, 828130561Sobrien driver_filter_t handler, void *arg) 82933965Sjdp{ 830130561Sobrien u_long vec; 83133965Sjdp int rid; 83233965Sjdp 83333965Sjdp rid = index; 83433965Sjdp sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev, 83533965Sjdp SYS_RES_IRQ, &rid, RF_ACTIVE); 83633965Sjdp if (sc->sc_irq_res[index] == NULL || 83733965Sjdp INTINO(vec = rman_get_start(sc->sc_irq_res[index])) != ino || 83833965Sjdp INTIGN(vec) != sc->sc_ign || 83933965Sjdp intr_vectors[vec].iv_ic != &fire_ic || 84033965Sjdp bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index], 84133965Sjdp INTR_TYPE_MISC | INTR_BRIDGE, handler, NULL, arg, 842130561Sobrien &sc->sc_ihand[index]) != 0) 84333965Sjdp panic("%s: failed to set up interrupt %d", __func__, index); 84433965Sjdp} 84533965Sjdp 84633965Sjdpstatic int 84733965Sjdpfire_intr_register(struct fire_softc *sc, u_int ino) 84833965Sjdp{ 84933965Sjdp struct fire_icarg *fica; 85033965Sjdp bus_addr_t intrclr, intrmap; 85133965Sjdp int error; 85233965Sjdp 85333965Sjdp if (fire_get_intrmap(sc, ino, &intrmap, &intrclr) == 0) 85433965Sjdp return (ENXIO); 85533965Sjdp fica = malloc((ino >= FO_EQ_FIRST_INO && ino <= FO_EQ_LAST_INO) ? 85633965Sjdp sizeof(struct fire_msiqarg) : sizeof(struct fire_icarg), M_DEVBUF, 85733965Sjdp M_NOWAIT | M_ZERO); 858130561Sobrien if (fica == NULL) 85933965Sjdp return (ENOMEM); 86033965Sjdp fica->fica_sc = sc; 86133965Sjdp fica->fica_map = intrmap; 86233965Sjdp fica->fica_clr = intrclr; 86333965Sjdp error = (intr_controller_register(INTMAP_VEC(sc->sc_ign, ino), 86433965Sjdp &fire_ic, fica)); 86533965Sjdp if (error != 0) 86633965Sjdp free(fica, M_DEVBUF); 86733965Sjdp return (error); 86833965Sjdp} 86933965Sjdp 87033965Sjdpstatic int 87133965Sjdpfire_get_intrmap(struct fire_softc *sc, u_int ino, bus_addr_t *intrmapptr, 87233965Sjdp bus_addr_t *intrclrptr) 87333965Sjdp{ 874130561Sobrien 87533965Sjdp if (ino > FO_MAX_INO) { 87633965Sjdp device_printf(sc->sc_dev, "out of range INO %d requested\n", 87733965Sjdp ino); 87833965Sjdp return (0); 87933965Sjdp } 88033965Sjdp 88133965Sjdp ino <<= 3; 88233965Sjdp if (intrmapptr != NULL) 88333965Sjdp *intrmapptr = FO_PCI_INT_MAP_BASE + ino; 88433965Sjdp if (intrclrptr != NULL) 88533965Sjdp *intrclrptr = FO_PCI_INT_CLR_BASE + ino; 886130561Sobrien return (1); 88733965Sjdp} 88833965Sjdp 889130561Sobrien/* 89033965Sjdp * Interrupt handlers 89133965Sjdp */ 89233965Sjdpstatic int 89333965Sjdpfire_dmc_pec(void *arg) 89433965Sjdp{ 89533965Sjdp struct fire_softc *sc; 89633965Sjdp device_t dev; 89733965Sjdp uint64_t cestat, dmcstat, ilustat, imustat, mcstat, mmustat, mmutfar; 89833965Sjdp uint64_t mmutfsr, oestat, pecstat, uestat, val; 89933965Sjdp u_int fatal, oenfatal; 90033965Sjdp 90133965Sjdp fatal = 0; 90233965Sjdp sc = arg; 90333965Sjdp dev = sc->sc_dev; 90433965Sjdp mtx_lock_spin(&sc->sc_pcib_mtx); 90533965Sjdp mcstat = FIRE_PCI_READ_8(sc, FO_PCI_MULTI_CORE_ERR_STAT); 90633965Sjdp if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_DMC) != 0) { 90733965Sjdp dmcstat = FIRE_PCI_READ_8(sc, FO_PCI_DMC_CORE_BLOCK_ERR_STAT); 90833965Sjdp if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_IMU) != 0) { 90933965Sjdp imustat = FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_STAT); 910130561Sobrien device_printf(dev, "IMU error %#llx\n", 91133965Sjdp (unsigned long long)imustat); 91233965Sjdp if ((imustat & 91333965Sjdp FO_PCI_IMU_ERR_INT_EQ_NOT_EN_P) != 0) { 91433965Sjdp fatal = 1; 915130561Sobrien val = FIRE_PCI_READ_8(sc, 91633965Sjdp FO_PCI_IMU_SCS_ERR_LOG); 91733965Sjdp device_printf(dev, "SCS error log %#llx\n", 918130561Sobrien (unsigned long long)val); 91933965Sjdp } 92033965Sjdp if ((imustat & FO_PCI_IMU_ERR_INT_EQ_OVER_P) != 0) { 92133965Sjdp fatal = 1; 92233965Sjdp val = FIRE_PCI_READ_8(sc, 92333965Sjdp FO_PCI_IMU_EQS_ERR_LOG); 92433965Sjdp device_printf(dev, "EQS error log %#llx\n", 92533965Sjdp (unsigned long long)val); 92633965Sjdp } 92733965Sjdp if ((imustat & (FO_PCI_IMU_ERR_INT_MSI_MAL_ERR_P | 92833965Sjdp FO_PCI_IMU_ERR_INT_MSI_PAR_ERR_P | 92933965Sjdp FO_PCI_IMU_ERR_INT_PMEACK_MES_NOT_EN_P | 93033965Sjdp FO_PCI_IMU_ERR_INT_PMPME_MES_NOT_EN_P | 93133965Sjdp FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P | 93233965Sjdp FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P | 93333965Sjdp FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P | 93433965Sjdp FO_PCI_IMU_ERR_INT_MSI_NOT_EN_P)) != 0) { 93533965Sjdp fatal = 1; 936130561Sobrien val = FIRE_PCI_READ_8(sc, 93733965Sjdp FO_PCI_IMU_RDS_ERR_LOG); 93833965Sjdp device_printf(dev, "RDS error log %#llx\n", 93933965Sjdp (unsigned long long)val); 94033965Sjdp } 94133965Sjdp } 94233965Sjdp if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_MMU) != 0) { 94333965Sjdp fatal = 1; 94433965Sjdp mmustat = FIRE_PCI_READ_8(sc, FO_PCI_MMU_INT_STAT); 94533965Sjdp mmutfar = FIRE_PCI_READ_8(sc, 946130561Sobrien FO_PCI_MMU_TRANS_FAULT_ADDR); 94733965Sjdp mmutfsr = FIRE_PCI_READ_8(sc, 94833965Sjdp FO_PCI_MMU_TRANS_FAULT_STAT); 949130561Sobrien if ((mmustat & (FO_PCI_MMU_ERR_INT_TBW_DPE_P | 95033965Sjdp FO_PCI_MMU_ERR_INT_TBW_ERR_P | 95133965Sjdp FO_PCI_MMU_ERR_INT_TBW_UDE_P | 95233965Sjdp FO_PCI_MMU_ERR_INT_TBW_DME_P | 95333965Sjdp FO_PCI_MMU_ERR_INT_TTC_CAE_P | 95433965Sjdp FIRE_PCI_MMU_ERR_INT_TTC_DPE_P | 95533965Sjdp OBERON_PCI_MMU_ERR_INT_TTC_DUE_P | 95633965Sjdp FO_PCI_MMU_ERR_INT_TRN_ERR_P)) != 0) 957130561Sobrien fatal = 1; 95833965Sjdp else { 95933965Sjdp sc->sc_stats_mmu_err++; 960130561Sobrien FIRE_PCI_WRITE_8(sc, FO_PCI_MMU_ERR_STAT_CLR, 96133965Sjdp mmustat); 96233965Sjdp } 96333965Sjdp device_printf(dev, 96433965Sjdp "MMU error %#llx: TFAR %#llx TFSR %#llx\n", 96533965Sjdp (unsigned long long)mmustat, 96633965Sjdp (unsigned long long)mmutfar, 967130561Sobrien (unsigned long long)mmutfsr); 96833965Sjdp } 96933965Sjdp } 97033965Sjdp if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_PEC) != 0) { 971130561Sobrien pecstat = FIRE_PCI_READ_8(sc, FO_PCI_PEC_CORE_BLOCK_INT_STAT); 97233965Sjdp if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_UERR) != 0) { 97333965Sjdp fatal = 1; 974130561Sobrien uestat = FIRE_PCI_READ_8(sc, 97533965Sjdp FO_PCI_TLU_UERR_INT_STAT); 97633965Sjdp device_printf(dev, 97733965Sjdp "DLU/TLU uncorrectable error %#llx\n", 97833965Sjdp (unsigned long long)uestat); 97933965Sjdp if ((uestat & (FO_PCI_TLU_UERR_INT_UR_P | 98033965Sjdp OBERON_PCI_TLU_UERR_INT_POIS_P | 981130561Sobrien FO_PCI_TLU_UERR_INT_MFP_P | 98233965Sjdp FO_PCI_TLU_UERR_INT_ROF_P | 98333965Sjdp FO_PCI_TLU_UERR_INT_UC_P | 98433965Sjdp FIRE_PCI_TLU_UERR_INT_PP_P | 985130561Sobrien OBERON_PCI_TLU_UERR_INT_POIS_P)) != 0) { 98633965Sjdp val = FIRE_PCI_READ_8(sc, 98733965Sjdp FO_PCI_TLU_RX_UERR_HDR1_LOG); 98833965Sjdp device_printf(dev, 98933965Sjdp "receive header log %#llx\n", 990130561Sobrien (unsigned long long)val); 99133965Sjdp val = FIRE_PCI_READ_8(sc, 99233965Sjdp FO_PCI_TLU_RX_UERR_HDR2_LOG); 99333965Sjdp device_printf(dev, 99433965Sjdp "receive header log 2 %#llx\n", 99533965Sjdp (unsigned long long)val); 99633965Sjdp } 99733965Sjdp if ((uestat & FO_PCI_TLU_UERR_INT_CTO_P) != 0) { 99833965Sjdp val = FIRE_PCI_READ_8(sc, 99933965Sjdp FO_PCI_TLU_TX_UERR_HDR1_LOG); 100033965Sjdp device_printf(dev, 100133965Sjdp "transmit header log %#llx\n", 1002130561Sobrien (unsigned long long)val); 1003130561Sobrien val = FIRE_PCI_READ_8(sc, 100433965Sjdp FO_PCI_TLU_TX_UERR_HDR2_LOG); 100533965Sjdp device_printf(dev, 100633965Sjdp "transmit header log 2 %#llx\n", 100733965Sjdp (unsigned long long)val); 1008130561Sobrien } 100933965Sjdp if ((uestat & FO_PCI_TLU_UERR_INT_DLP_P) != 0) { 101033965Sjdp val = FIRE_PCI_READ_8(sc, 101133965Sjdp FO_PCI_LPU_LNK_LYR_INT_STAT); 101233965Sjdp device_printf(dev, 101333965Sjdp "link layer interrupt and status %#llx\n", 1014130561Sobrien (unsigned long long)val); 101533965Sjdp } 1016130561Sobrien if ((uestat & FO_PCI_TLU_UERR_INT_TE_P) != 0) { 101733965Sjdp val = FIRE_PCI_READ_8(sc, 101860484Sobrien FO_PCI_LPU_PHY_LYR_INT_STAT); 101960484Sobrien device_printf(dev, 102060484Sobrien "phy layer interrupt and status %#llx\n", 102160484Sobrien (unsigned long long)val); 102260484Sobrien } 102333965Sjdp } 102433965Sjdp if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_CERR) != 0) { 1025130561Sobrien sc->sc_stats_tlu_ce++; 102633965Sjdp cestat = FIRE_PCI_READ_8(sc, 102733965Sjdp FO_PCI_TLU_CERR_INT_STAT); 102833965Sjdp device_printf(dev, 102933965Sjdp "DLU/TLU correctable error %#llx\n", 103060484Sobrien (unsigned long long)cestat); 103160484Sobrien val = FIRE_PCI_READ_8(sc, 103260484Sobrien FO_PCI_LPU_LNK_LYR_INT_STAT); 103360484Sobrien device_printf(dev, 103460484Sobrien "link layer interrupt and status %#llx\n", 103533965Sjdp (unsigned long long)val); 103660484Sobrien if ((cestat & FO_PCI_TLU_CERR_INT_RE_P) != 0) { 103760484Sobrien FIRE_PCI_WRITE_8(sc, 103860484Sobrien FO_PCI_LPU_LNK_LYR_INT_STAT, val); 103960484Sobrien val = FIRE_PCI_READ_8(sc, 104060484Sobrien FO_PCI_LPU_PHY_LYR_INT_STAT); 104160484Sobrien device_printf(dev, 104260484Sobrien "phy layer interrupt and status %#llx\n", 104360484Sobrien (unsigned long long)val); 104460484Sobrien } 104560484Sobrien FIRE_PCI_WRITE_8(sc, FO_PCI_TLU_CERR_STAT_CLR, 104660484Sobrien cestat); 104733965Sjdp } 104833965Sjdp if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_OEVENT) != 0) { 104933965Sjdp oenfatal = 0; 105033965Sjdp oestat = FIRE_PCI_READ_8(sc, 105133965Sjdp FO_PCI_TLU_OEVENT_INT_STAT); 1052130561Sobrien device_printf(dev, "DLU/TLU other event %#llx\n", 105333965Sjdp (unsigned long long)oestat); 105433965Sjdp if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 105533965Sjdp FO_PCI_TLU_OEVENT_MRC_P | 105633965Sjdp FO_PCI_TLU_OEVENT_WUC_P | 105733965Sjdp FO_PCI_TLU_OEVENT_RUC_P | 105833965Sjdp FO_PCI_TLU_OEVENT_CRS_P)) != 0) { 105933965Sjdp val = FIRE_PCI_READ_8(sc, 106033965Sjdp FO_PCI_TLU_RX_OEVENT_HDR1_LOG); 106133965Sjdp device_printf(dev, 106233965Sjdp "receive header log %#llx\n", 106333965Sjdp (unsigned long long)val); 106433965Sjdp val = FIRE_PCI_READ_8(sc, 1065130561Sobrien FO_PCI_TLU_RX_OEVENT_HDR2_LOG); 106633965Sjdp device_printf(dev, 106733965Sjdp "receive header log 2 %#llx\n", 106833965Sjdp (unsigned long long)val); 1069130561Sobrien if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 107033965Sjdp FO_PCI_TLU_OEVENT_MRC_P | 107133965Sjdp FO_PCI_TLU_OEVENT_WUC_P | 107233965Sjdp FO_PCI_TLU_OEVENT_RUC_P)) != 0) 107333965Sjdp fatal = 1; 107433965Sjdp else { 107533965Sjdp sc->sc_stats_tlu_oe_rx_err++; 107633965Sjdp oenfatal = 1; 1077130561Sobrien } 107833965Sjdp } 107933965Sjdp if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 1080130561Sobrien FO_PCI_TLU_OEVENT_CTO_P | 108133965Sjdp FO_PCI_TLU_OEVENT_WUC_P | 108233965Sjdp FO_PCI_TLU_OEVENT_RUC_P)) != 0) { 108333965Sjdp val = FIRE_PCI_READ_8(sc, 108433965Sjdp FO_PCI_TLU_TX_OEVENT_HDR1_LOG); 108533965Sjdp device_printf(dev, 108633965Sjdp "transmit header log %#llx\n", 108733965Sjdp (unsigned long long)val); 1088130561Sobrien val = FIRE_PCI_READ_8(sc, 108933965Sjdp FO_PCI_TLU_TX_OEVENT_HDR2_LOG); 109033965Sjdp device_printf(dev, 1091130561Sobrien "transmit header log 2 %#llx\n", 109233965Sjdp (unsigned long long)val); 109333965Sjdp if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | 109433965Sjdp FO_PCI_TLU_OEVENT_CTO_P | 109533965Sjdp FO_PCI_TLU_OEVENT_WUC_P | 109633965Sjdp FO_PCI_TLU_OEVENT_RUC_P)) != 0) 109733965Sjdp fatal = 1; 109833965Sjdp else { 109933965Sjdp sc->sc_stats_tlu_oe_tx_err++; 110033965Sjdp oenfatal = 1; 110133965Sjdp } 1102130561Sobrien } 110333965Sjdp if ((oestat & (FO_PCI_TLU_OEVENT_ERO_P | 110433965Sjdp FO_PCI_TLU_OEVENT_EMP_P | 1105130561Sobrien FO_PCI_TLU_OEVENT_EPE_P | 110633965Sjdp FIRE_PCI_TLU_OEVENT_ERP_P | 110733965Sjdp OBERON_PCI_TLU_OEVENT_ERBU_P | 110833965Sjdp FIRE_PCI_TLU_OEVENT_EIP_P | 110933965Sjdp OBERON_PCI_TLU_OEVENT_EIUE_P)) != 0) { 1110130561Sobrien fatal = 1; 111133965Sjdp val = FIRE_PCI_READ_8(sc, 111233965Sjdp FO_PCI_LPU_LNK_LYR_INT_STAT); 111333965Sjdp device_printf(dev, 111433965Sjdp "link layer interrupt and status %#llx\n", 111533965Sjdp (unsigned long long)val); 1116130561Sobrien } 111733965Sjdp if ((oestat & (FO_PCI_TLU_OEVENT_IIP_P | 111833965Sjdp FO_PCI_TLU_OEVENT_EDP_P | 111933965Sjdp FIRE_PCI_TLU_OEVENT_EHP_P | 112033965Sjdp OBERON_PCI_TLU_OEVENT_TLUEITMO_S | 112133965Sjdp FO_PCI_TLU_OEVENT_ERU_P)) != 0) 112233965Sjdp fatal = 1; 112333965Sjdp if ((oestat & (FO_PCI_TLU_OEVENT_NFP_P | 112433965Sjdp FO_PCI_TLU_OEVENT_LWC_P | 1125130561Sobrien FO_PCI_TLU_OEVENT_LIN_P | 112633965Sjdp FO_PCI_TLU_OEVENT_LRS_P | 112733965Sjdp FO_PCI_TLU_OEVENT_LDN_P | 112833965Sjdp FO_PCI_TLU_OEVENT_LUP_P)) != 0) 112933965Sjdp oenfatal = 1; 1130130561Sobrien if (oenfatal != 0) { 113133965Sjdp sc->sc_stats_tlu_oe_non_fatal++; 113233965Sjdp FIRE_PCI_WRITE_8(sc, 113333965Sjdp FO_PCI_TLU_OEVENT_STAT_CLR, oestat); 113433965Sjdp if ((oestat & FO_PCI_TLU_OEVENT_LIN_P) != 0) 113533965Sjdp FIRE_PCI_WRITE_8(sc, 113633965Sjdp FO_PCI_LPU_LNK_LYR_INT_STAT, 113733965Sjdp FIRE_PCI_READ_8(sc, 113833965Sjdp FO_PCI_LPU_LNK_LYR_INT_STAT)); 113933965Sjdp } 1140130561Sobrien } 114133965Sjdp if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_ILU) != 0) { 1142130561Sobrien ilustat = FIRE_PCI_READ_8(sc, FO_PCI_ILU_INT_STAT); 114360484Sobrien device_printf(dev, "ILU error %#llx\n", 114433965Sjdp (unsigned long long)ilustat); 114533965Sjdp if ((ilustat & (FIRE_PCI_ILU_ERR_INT_IHB_PE_P | 114633965Sjdp FIRE_PCI_ILU_ERR_INT_IHB_PE_P)) != 0) 114789857Sobrien fatal = 1; 114833965Sjdp else { 114933965Sjdp sc->sc_stats_ilu_err++; 115033965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_ILU_INT_STAT, 115133965Sjdp ilustat); 115233965Sjdp } 115333965Sjdp } 115433965Sjdp } 115533965Sjdp mtx_unlock_spin(&sc->sc_pcib_mtx); 115633965Sjdp if (fatal != 0) 115733965Sjdp panic("%s: fatal DMC/PEC error", 115833965Sjdp device_get_nameunit(sc->sc_dev)); 1159130561Sobrien return (FILTER_HANDLED); 1160130561Sobrien} 1161130561Sobrien 116233965Sjdpstatic int 116333965Sjdpfire_xcb(void *arg) 116433965Sjdp{ 116533965Sjdp struct fire_softc *sc; 116633965Sjdp device_t dev; 116733965Sjdp uint64_t errstat, intstat, val; 116833965Sjdp u_int fatal; 116933965Sjdp 117033965Sjdp fatal = 0; 117133965Sjdp sc = arg; 117233965Sjdp dev = sc->sc_dev; 117333965Sjdp mtx_lock_spin(&sc->sc_pcib_mtx); 117433965Sjdp if (sc->sc_mode == FIRE_MODE_OBERON) { 117533965Sjdp intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT); 117633965Sjdp device_printf(dev, "UBC error: interrupt status %#llx\n", 117733965Sjdp (unsigned long long)intstat); 117833965Sjdp if ((intstat & ~(OBERON_UBC_ERR_INT_DMARDUEB_P | 117933965Sjdp OBERON_UBC_ERR_INT_DMARDUEA_P)) != 0) 118033965Sjdp fatal = 1; 118133965Sjdp else 118233965Sjdp sc->sc_stats_ubc_dmardue++; 118333965Sjdp if (fatal != 0) { 118433965Sjdp mtx_unlock_spin(&sc->sc_pcib_mtx); 118533965Sjdp panic("%s: fatal UBC core block error", 118633965Sjdp device_get_nameunit(sc->sc_dev)); 118733965Sjdp } else { 118889857Sobrien FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); 1189130561Sobrien mtx_unlock_spin(&sc->sc_pcib_mtx); 1190130561Sobrien } 119133965Sjdp } else { 119233965Sjdp errstat = FIRE_CTRL_READ_8(sc, FIRE_JBC_CORE_BLOCK_ERR_STAT); 119333965Sjdp if ((errstat & (FIRE_JBC_CORE_BLOCK_ERR_STAT_MERGE | 119433965Sjdp FIRE_JBC_CORE_BLOCK_ERR_STAT_JBCINT | 119533965Sjdp FIRE_JBC_CORE_BLOCK_ERR_STAT_DMCINT)) != 0) { 119633965Sjdp intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT); 119733965Sjdp device_printf(dev, "JBC interrupt status %#llx\n", 119833965Sjdp (unsigned long long)intstat); 119933965Sjdp if ((intstat & FIRE_JBC_ERR_INT_EBUS_TO_P) != 0) { 120033965Sjdp val = FIRE_CTRL_READ_8(sc, 120133965Sjdp FIRE_JBC_CSR_ERR_LOG); 120233965Sjdp device_printf(dev, "CSR error log %#llx\n", 120333965Sjdp (unsigned long long)val); 120433965Sjdp } 120533965Sjdp if ((intstat & (FIRE_JBC_ERR_INT_UNSOL_RD_P | 120633965Sjdp FIRE_JBC_ERR_INT_UNSOL_INT_P)) != 0) { 120733965Sjdp if ((intstat & 120833965Sjdp FIRE_JBC_ERR_INT_UNSOL_RD_P) != 0) 120968765Sobrien sc->sc_stats_jbc_unsol_rd++; 121033965Sjdp if ((intstat & 121133965Sjdp FIRE_JBC_ERR_INT_UNSOL_INT_P) != 0) 121233965Sjdp sc->sc_stats_jbc_unsol_int++; 121333965Sjdp val = FIRE_CTRL_READ_8(sc, 121433965Sjdp FIRE_DMCINT_IDC_ERR_LOG); 1215130561Sobrien device_printf(dev, 121633965Sjdp "DMCINT IDC error log %#llx\n", 121733965Sjdp (unsigned long long)val); 121833965Sjdp } 121933965Sjdp if ((intstat & (FIRE_JBC_ERR_INT_MB_PER_P | 122033965Sjdp FIRE_JBC_ERR_INT_MB_PEW_P)) != 0) { 122133965Sjdp fatal = 1; 122233965Sjdp val = FIRE_CTRL_READ_8(sc, 122333965Sjdp FIRE_MERGE_TRANS_ERR_LOG); 122433965Sjdp device_printf(dev, 122533965Sjdp "merge transaction error log %#llx\n", 122633965Sjdp (unsigned long long)val); 122733965Sjdp } 122833965Sjdp if ((intstat & FIRE_JBC_ERR_INT_IJP_P) != 0) { 122933965Sjdp fatal = 1; 123033965Sjdp val = FIRE_CTRL_READ_8(sc, 123133965Sjdp FIRE_JBCINT_OTRANS_ERR_LOG); 123233965Sjdp device_printf(dev, 123333965Sjdp "JBCINT out transaction error log " 123433965Sjdp "%#llx\n", (unsigned long long)val); 123533965Sjdp val = FIRE_CTRL_READ_8(sc, 123633965Sjdp FIRE_JBCINT_OTRANS_ERR_LOG2); 123733965Sjdp device_printf(dev, 123833965Sjdp "JBCINT out transaction error log 2 " 123933965Sjdp "%#llx\n", (unsigned long long)val); 124033965Sjdp } 124133965Sjdp if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P | 124233965Sjdp FIRE_JBC_ERR_INT_CE_ASYN_P | 124333965Sjdp FIRE_JBC_ERR_INT_JTE_P | FIRE_JBC_ERR_INT_JBE_P | 124433965Sjdp FIRE_JBC_ERR_INT_JUE_P | 124533965Sjdp FIRE_JBC_ERR_INT_ICISE_P | 124633965Sjdp FIRE_JBC_ERR_INT_WR_DPE_P | 124733965Sjdp FIRE_JBC_ERR_INT_RD_DPE_P | 124833965Sjdp FIRE_JBC_ERR_INT_ILL_BMW_P | 124933965Sjdp FIRE_JBC_ERR_INT_ILL_BMR_P | 125033965Sjdp FIRE_JBC_ERR_INT_BJC_P)) != 0) { 125160484Sobrien if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P | 125233965Sjdp FIRE_JBC_ERR_INT_JTE_P | 125333965Sjdp FIRE_JBC_ERR_INT_JBE_P | 125433965Sjdp FIRE_JBC_ERR_INT_JUE_P | 125533965Sjdp FIRE_JBC_ERR_INT_ICISE_P | 125633965Sjdp FIRE_JBC_ERR_INT_WR_DPE_P | 125733965Sjdp FIRE_JBC_ERR_INT_RD_DPE_P | 125833965Sjdp FIRE_JBC_ERR_INT_ILL_BMW_P | 125933965Sjdp FIRE_JBC_ERR_INT_ILL_BMR_P | 126033965Sjdp FIRE_JBC_ERR_INT_BJC_P)) != 0) 126133965Sjdp fatal = 1; 126233965Sjdp else 126333965Sjdp sc->sc_stats_jbc_ce_async++; 126460484Sobrien val = FIRE_CTRL_READ_8(sc, 126533965Sjdp FIRE_JBCINT_ITRANS_ERR_LOG); 126660484Sobrien device_printf(dev, 126760484Sobrien "JBCINT in transaction error log %#llx\n", 126860484Sobrien (unsigned long long)val); 126933965Sjdp val = FIRE_CTRL_READ_8(sc, 127060484Sobrien FIRE_JBCINT_ITRANS_ERR_LOG2); 127160484Sobrien device_printf(dev, 127260484Sobrien "JBCINT in transaction error log 2 " 127360484Sobrien "%#llx\n", (unsigned long long)val); 127460484Sobrien } 127560484Sobrien if ((intstat & (FIRE_JBC_ERR_INT_PIO_UNMAP_RD_P | 127660484Sobrien FIRE_JBC_ERR_INT_ILL_ACC_RD_P | 127760484Sobrien FIRE_JBC_ERR_INT_PIO_UNMAP_P | 127860484Sobrien FIRE_JBC_ERR_INT_PIO_DPE_P | 127960484Sobrien FIRE_JBC_ERR_INT_PIO_CPE_P | 128033965Sjdp FIRE_JBC_ERR_INT_ILL_ACC_P)) != 0) { 128133965Sjdp fatal = 1; 128233965Sjdp val = FIRE_CTRL_READ_8(sc, 128333965Sjdp FIRE_JBC_CSR_ERR_LOG); 128433965Sjdp device_printf(dev, 128560484Sobrien "DMCINT ODCD error log %#llx\n", 128660484Sobrien (unsigned long long)val); 128760484Sobrien } 128860484Sobrien if ((intstat & (FIRE_JBC_ERR_INT_MB_PEA_P | 128960484Sobrien FIRE_JBC_ERR_INT_CPE_P | FIRE_JBC_ERR_INT_APE_P | 129060484Sobrien FIRE_JBC_ERR_INT_PIO_CPE_P | 129160484Sobrien FIRE_JBC_ERR_INT_JTCEEW_P | 1292130561Sobrien FIRE_JBC_ERR_INT_JTCEEI_P | 129360484Sobrien FIRE_JBC_ERR_INT_JTCEER_P)) != 0) { 129433965Sjdp fatal = 1; 129533965Sjdp val = FIRE_CTRL_READ_8(sc, 129633965Sjdp FIRE_FATAL_ERR_LOG); 129733965Sjdp device_printf(dev, "fatal error log %#llx\n", 129833965Sjdp (unsigned long long)val); 129933965Sjdp val = FIRE_CTRL_READ_8(sc, 130033965Sjdp FIRE_FATAL_ERR_LOG2); 130133965Sjdp device_printf(dev, "fatal error log 2 " 130233965Sjdp "%#llx\n", (unsigned long long)val); 130333965Sjdp } 130433965Sjdp if (fatal != 0) { 130533965Sjdp mtx_unlock_spin(&sc->sc_pcib_mtx); 130633965Sjdp panic("%s: fatal JBC core block error", 130733965Sjdp device_get_nameunit(sc->sc_dev)); 130833965Sjdp } else { 130933965Sjdp FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); 131033965Sjdp mtx_unlock_spin(&sc->sc_pcib_mtx); 131133965Sjdp } 131233965Sjdp } else { 131333965Sjdp mtx_unlock_spin(&sc->sc_pcib_mtx); 131433965Sjdp panic("%s: unknown JCB core block error status %#llx", 131533965Sjdp device_get_nameunit(sc->sc_dev), 131633965Sjdp (unsigned long long)errstat); 131733965Sjdp } 131833965Sjdp } 131933965Sjdp return (FILTER_HANDLED); 132033965Sjdp} 132133965Sjdp 132233965Sjdpstatic int 132333965Sjdpfire_pcie(void *arg) 132433965Sjdp{ 132533965Sjdp struct fire_msiqarg *fmqa; 132633965Sjdp struct fire_softc *sc; 132733965Sjdp struct fo_msiq_record *qrec; 132833965Sjdp device_t dev; 132933965Sjdp uint64_t word0; 133033965Sjdp u_int head, msg, msiq; 133133965Sjdp 133233965Sjdp fmqa = arg; 133333965Sjdp sc = fmqa->fmqa_fica.fica_sc; 133433965Sjdp dev = sc->sc_dev; 133533965Sjdp msiq = fmqa->fmqa_msiq; 133633965Sjdp mtx_lock_spin(&fmqa->fmqa_mtx); 133733965Sjdp head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >> 133833965Sjdp FO_PCI_EQ_HD_SHFT; 133933965Sjdp qrec = &fmqa->fmqa_base[head]; 134033965Sjdp word0 = qrec->fomqr_word0; 134133965Sjdp for (;;) { 134233965Sjdp KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSG) != 0, 134333965Sjdp ("%s: received non-PCIe message in event queue %d " 134433965Sjdp "(word0 %#llx)", device_get_nameunit(dev), msiq, 134533965Sjdp (unsigned long long)word0)); 134633965Sjdp msg = (word0 & FO_MQR_WORD0_DATA0_MASK) >> 134733965Sjdp FO_MQR_WORD0_DATA0_SHFT; 134833965Sjdp 134933965Sjdp#define PCIE_MSG_CODE_ERR_COR 0x30 135033965Sjdp#define PCIE_MSG_CODE_ERR_NONFATAL 0x31 135133965Sjdp#define PCIE_MSG_CODE_ERR_FATAL 0x33 135233965Sjdp 135333965Sjdp if (msg == PCIE_MSG_CODE_ERR_COR) 135433965Sjdp device_printf(dev, "correctable PCIe error\n"); 135533965Sjdp else if (msg == PCIE_MSG_CODE_ERR_NONFATAL || 135633965Sjdp msg == PCIE_MSG_CODE_ERR_FATAL) 135733965Sjdp panic("%s: %sfatal PCIe error", 135833965Sjdp device_get_nameunit(dev), 135933965Sjdp msg == PCIE_MSG_CODE_ERR_NONFATAL ? "non-" : ""); 136033965Sjdp else 136133965Sjdp panic("%s: received unknown PCIe message %#x", 136233965Sjdp device_get_nameunit(dev), msg); 136333965Sjdp qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK; 136433965Sjdp head = (head + 1) % sc->sc_msiq_size; 136533965Sjdp qrec = &fmqa->fmqa_base[head]; 136633965Sjdp word0 = qrec->fomqr_word0; 136733965Sjdp if (__predict_true((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0)) 136833965Sjdp break; 136933965Sjdp } 137033965Sjdp FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) << 137133965Sjdp FO_PCI_EQ_HD_SHFT); 137233965Sjdp if ((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) & 137333965Sjdp FO_PCI_EQ_TL_OVERR) != 0) { 137433965Sjdp device_printf(dev, "event queue %d overflow\n", msiq); 1375130561Sobrien msiq <<= 3; 137633965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, 137733965Sjdp FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) | 137833965Sjdp FO_PCI_EQ_CTRL_CLR_COVERR); 137933965Sjdp } 138033965Sjdp mtx_unlock_spin(&fmqa->fmqa_mtx); 138133965Sjdp return (FILTER_HANDLED); 138233965Sjdp} 138333965Sjdp 138433965Sjdpstatic int 138533965Sjdpfire_maxslots(device_t dev) 138633965Sjdp{ 138733965Sjdp 138833965Sjdp return (1); 138933965Sjdp} 139033965Sjdp 139133965Sjdpstatic uint32_t 139233965Sjdpfire_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, 139333965Sjdp int width) 139433965Sjdp{ 139533965Sjdp struct fire_softc *sc; 139633965Sjdp bus_space_handle_t bh; 139733965Sjdp u_long offset = 0; 139833965Sjdp uint32_t r, wrd; 139933965Sjdp int i; 140033965Sjdp uint16_t shrt; 140133965Sjdp uint8_t byte; 140233965Sjdp 140333965Sjdp sc = device_get_softc(dev); 140433965Sjdp if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus || 140533965Sjdp slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCIE_REGMAX) 140633965Sjdp return (-1); 140733965Sjdp 140833965Sjdp offset = FO_CONF_OFF(bus, slot, func, reg); 140933965Sjdp bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG]; 141033965Sjdp switch (width) { 141133965Sjdp case 1: 141233965Sjdp i = bus_space_peek_1(sc->sc_pci_cfgt, bh, offset, &byte); 141333965Sjdp r = byte; 141433965Sjdp break; 141533965Sjdp case 2: 141633965Sjdp i = bus_space_peek_2(sc->sc_pci_cfgt, bh, offset, &shrt); 141733965Sjdp r = shrt; 141833965Sjdp break; 141933965Sjdp case 4: 142033965Sjdp i = bus_space_peek_4(sc->sc_pci_cfgt, bh, offset, &wrd); 142133965Sjdp r = wrd; 142233965Sjdp break; 142333965Sjdp default: 142433965Sjdp panic("%s: bad width", __func__); 142533965Sjdp /* NOTREACHED */ 142633965Sjdp } 142733965Sjdp 142833965Sjdp if (i) { 142933965Sjdp#ifdef FIRE_DEBUG 143033965Sjdp printf("%s: read data error reading: %d.%d.%d: 0x%x\n", 143133965Sjdp __func__, bus, slot, func, reg); 143233965Sjdp#endif 143333965Sjdp r = -1; 143433965Sjdp } 143533965Sjdp return (r); 143633965Sjdp} 143733965Sjdp 143833965Sjdpstatic void 143933965Sjdpfire_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, 144033965Sjdp uint32_t val, int width) 144133965Sjdp{ 144233965Sjdp struct fire_softc *sc; 144333965Sjdp bus_space_handle_t bh; 144433965Sjdp u_long offset = 0; 144533965Sjdp 1446130561Sobrien sc = device_get_softc(dev); 144733965Sjdp if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus || 144833965Sjdp slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCIE_REGMAX) 144933965Sjdp return; 145033965Sjdp 145133965Sjdp offset = FO_CONF_OFF(bus, slot, func, reg); 145233965Sjdp bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG]; 145333965Sjdp switch (width) { 145433965Sjdp case 1: 1455130561Sobrien bus_space_write_1(sc->sc_pci_cfgt, bh, offset, val); 145633965Sjdp break; 145733965Sjdp case 2: 145833965Sjdp bus_space_write_2(sc->sc_pci_cfgt, bh, offset, val); 145933965Sjdp break; 146033965Sjdp case 4: 146133965Sjdp bus_space_write_4(sc->sc_pci_cfgt, bh, offset, val); 146233965Sjdp break; 146333965Sjdp default: 146433965Sjdp panic("%s: bad width", __func__); 146533965Sjdp /* NOTREACHED */ 146633965Sjdp } 146733965Sjdp} 146833965Sjdp 146933965Sjdpstatic int 147033965Sjdpfire_route_interrupt(device_t bridge, device_t dev, int pin) 147133965Sjdp{ 147233965Sjdp struct fire_softc *sc; 147333965Sjdp struct ofw_pci_register reg; 147433965Sjdp ofw_pci_intr_t pintr, mintr; 147533965Sjdp 147633965Sjdp sc = device_get_softc(bridge); 147733965Sjdp pintr = pin; 147833965Sjdp if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo, 147933965Sjdp ®, sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr), 148033965Sjdp NULL) != 0) 148133965Sjdp return (mintr); 148233965Sjdp 148333965Sjdp device_printf(bridge, "could not route pin %d for device %d.%d\n", 148433965Sjdp pin, pci_get_slot(dev), pci_get_function(dev)); 148533965Sjdp return (PCI_INVALID_IRQ); 148633965Sjdp} 148733965Sjdp 148833965Sjdpstatic int 148933965Sjdpfire_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) 1490130561Sobrien{ 149133965Sjdp struct fire_softc *sc; 149233965Sjdp 149333965Sjdp sc = device_get_softc(dev); 149433965Sjdp switch (which) { 149533965Sjdp case PCIB_IVAR_DOMAIN: 149633965Sjdp *result = device_get_unit(dev); 149733965Sjdp return (0); 149833965Sjdp case PCIB_IVAR_BUS: 149933965Sjdp *result = sc->sc_pci_secbus; 150033965Sjdp return (0); 150133965Sjdp } 150233965Sjdp return (ENOENT); 150333965Sjdp} 150433965Sjdp 150533965Sjdpstatic void 1506130561Sobrienfire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map, 150733965Sjdp bus_dmasync_op_t op) 150833965Sjdp{ 150933965Sjdp static u_char buf[VIS_BLOCKSIZE] __aligned(VIS_BLOCKSIZE); 1510130561Sobrien register_t reg, s; 151133965Sjdp 151233965Sjdp if ((map->dm_flags & DMF_LOADED) == 0) 151333965Sjdp return; 151433965Sjdp 151533965Sjdp if ((op & BUS_DMASYNC_POSTREAD) != 0) { 151633965Sjdp s = intr_disable(); 151733965Sjdp reg = rd(fprs); 151833965Sjdp wr(fprs, reg | FPRS_FEF, 0); 151933965Sjdp __asm __volatile("stda %%f0, [%0] %1" 152033965Sjdp : : "r" (buf), "n" (ASI_BLK_COMMIT_S)); 152133965Sjdp membar(Sync); 152233965Sjdp wr(fprs, reg, 0); 152333965Sjdp intr_restore(s); 152433965Sjdp } else if ((op & BUS_DMASYNC_PREWRITE) != 0) 152533965Sjdp membar(Sync); 152633965Sjdp} 152733965Sjdp 152833965Sjdpstatic void 152933965Sjdpfire_intr_enable(void *arg) 153033965Sjdp{ 153133965Sjdp struct intr_vector *iv; 153233965Sjdp struct fire_icarg *fica; 153333965Sjdp struct fire_softc *sc; 153433965Sjdp struct pcpu *pc; 153533965Sjdp uint64_t mr; 153633965Sjdp u_int ctrl, i; 153733965Sjdp 153833965Sjdp iv = arg; 153933965Sjdp fica = iv->iv_icarg; 154033965Sjdp sc = fica->fica_sc; 154133965Sjdp mr = FO_PCI_IMAP_V; 154233965Sjdp if (sc->sc_mode == FIRE_MODE_OBERON) 154333965Sjdp mr |= (iv->iv_mid << OBERON_PCI_IMAP_T_DESTID_SHFT) & 154433965Sjdp OBERON_PCI_IMAP_T_DESTID_MASK; 154533965Sjdp else 154633965Sjdp mr |= (iv->iv_mid << FIRE_PCI_IMAP_T_JPID_SHFT) & 154733965Sjdp FIRE_PCI_IMAP_T_JPID_MASK; 154833965Sjdp /* 154933965Sjdp * Given that all mondos for the same target are required to use the 155033965Sjdp * same interrupt controller we just use the CPU ID for indexing the 155133965Sjdp * latter. 155233965Sjdp */ 155333965Sjdp ctrl = 0; 155433965Sjdp for (i = 0; i < mp_ncpus; ++i) { 155533965Sjdp pc = pcpu_find(i); 155633965Sjdp if (pc == NULL || iv->iv_mid != pc->pc_mid) 155733965Sjdp continue; 155833965Sjdp ctrl = pc->pc_cpuid % 4; 155933965Sjdp break; 156033965Sjdp } 156133965Sjdp mr |= (1ULL << ctrl) << FO_PCI_IMAP_INT_CTRL_NUM_SHFT & 156233965Sjdp FO_PCI_IMAP_INT_CTRL_NUM_MASK; 156333965Sjdp FIRE_PCI_WRITE_8(sc, fica->fica_map, mr); 156433965Sjdp} 156533965Sjdp 156633965Sjdpstatic void 156733965Sjdpfire_intr_disable(void *arg) 156833965Sjdp{ 156933965Sjdp struct intr_vector *iv; 157033965Sjdp struct fire_icarg *fica; 157133965Sjdp struct fire_softc *sc; 157233965Sjdp 157333965Sjdp iv = arg; 157433965Sjdp fica = iv->iv_icarg; 157533965Sjdp sc = fica->fica_sc; 157633965Sjdp FIRE_PCI_WRITE_8(sc, fica->fica_map, 157733965Sjdp FIRE_PCI_READ_8(sc, fica->fica_map) & ~FO_PCI_IMAP_V); 157833965Sjdp} 157933965Sjdp 158033965Sjdpstatic void 158133965Sjdpfire_intr_assign(void *arg) 158233965Sjdp{ 158333965Sjdp struct intr_vector *iv; 158433965Sjdp struct fire_icarg *fica; 158533965Sjdp struct fire_softc *sc; 158660484Sobrien uint64_t mr; 158733965Sjdp 158833965Sjdp iv = arg; 158933965Sjdp fica = iv->iv_icarg; 159033965Sjdp sc = fica->fica_sc; 159133965Sjdp mr = FIRE_PCI_READ_8(sc, fica->fica_map); 159233965Sjdp if ((mr & FO_PCI_IMAP_V) != 0) { 159333965Sjdp FIRE_PCI_WRITE_8(sc, fica->fica_map, mr & ~FO_PCI_IMAP_V); 159433965Sjdp FIRE_PCI_BARRIER(sc, fica->fica_map, 8, 159533965Sjdp BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 159633965Sjdp } 1597130561Sobrien while (FIRE_PCI_READ_8(sc, fica->fica_clr) != INTCLR_IDLE) 1598130561Sobrien ; 159933965Sjdp if ((mr & FO_PCI_IMAP_V) != 0) 160033965Sjdp fire_intr_enable(arg); 160133965Sjdp} 160233965Sjdp 160333965Sjdpstatic void 160433965Sjdpfire_intr_clear(void *arg) 160533965Sjdp{ 160633965Sjdp struct intr_vector *iv; 1607130561Sobrien struct fire_icarg *fica; 160833965Sjdp 160933965Sjdp iv = arg; 161033965Sjdp fica = iv->iv_icarg; 161133965Sjdp FIRE_PCI_WRITE_8(fica->fica_sc, fica->fica_clr, INTCLR_IDLE); 1612130561Sobrien} 161333965Sjdp 161433965Sjdp/* 161533965Sjdp * Given that the event queue implementation matches our current MD and MI 1616130561Sobrien * interrupt frameworks like square pegs fit into round holes we are generous 161733965Sjdp * and use one event queue per MSI for now, which limits us to 35 MSIs/MSI-Xs 161833965Sjdp * per Host-PCIe-bridge (we use one event queue for the PCIe error messages). 1619130561Sobrien * This seems tolerable as long as most devices just use one MSI/MSI-X anyway. 162033965Sjdp * Adding knowledge about MSIs/MSI-Xs to the MD interrupt code should allow us 162133965Sjdp * to decouple the 1:1 mapping at the cost of no longer being able to bind 162233965Sjdp * MSIs/MSI-Xs to specific CPUs as we currently have no reliable way to 1623130561Sobrien * quiesce a device while we move its MSIs/MSI-Xs to another event queue. 162433965Sjdp */ 162533965Sjdp 162633965Sjdpstatic int 162733965Sjdpfire_alloc_msi(device_t dev, device_t child, int count, int maxcount __unused, 1628130561Sobrien int *irqs) 162933965Sjdp{ 163033965Sjdp struct fire_softc *sc; 163133965Sjdp u_int i, j, msiqrun; 163233965Sjdp 163333965Sjdp if (powerof2(count) == 0 || count > 32) 1634130561Sobrien return (EINVAL); 163533965Sjdp 163633965Sjdp sc = device_get_softc(dev); 163733965Sjdp mtx_lock(&sc->sc_msi_mtx); 1638130561Sobrien msiqrun = 0; 163933965Sjdp for (i = 0; i < sc->sc_msiq_count; i++) { 164033965Sjdp for (j = i; j < i + count; j++) { 164133965Sjdp if (isclr(sc->sc_msiq_bitmap, j) == 0) 1642130561Sobrien break; 164333965Sjdp } 164433965Sjdp if (j == i + count) { 164533965Sjdp msiqrun = i; 164633965Sjdp break; 164733965Sjdp } 164833965Sjdp } 164933965Sjdp if (i == sc->sc_msiq_count) { 165033965Sjdp mtx_unlock(&sc->sc_msi_mtx); 165133965Sjdp return (ENXIO); 165233965Sjdp } 165333965Sjdp for (i = 0; i + count < sc->sc_msi_count; i += count) { 165433965Sjdp for (j = i; j < i + count; j++) 165533965Sjdp if (isclr(sc->sc_msi_bitmap, j) == 0) 165633965Sjdp break; 165733965Sjdp if (j == i + count) { 165833965Sjdp for (j = 0; j < count; j++) { 165933965Sjdp setbit(sc->sc_msiq_bitmap, msiqrun + j); 166033965Sjdp setbit(sc->sc_msi_bitmap, i + j); 166133965Sjdp sc->sc_msi_msiq_table[i + j] = msiqrun + j; 166233965Sjdp irqs[j] = sc->sc_msi_first + i + j; 166333965Sjdp } 166433965Sjdp mtx_unlock(&sc->sc_msi_mtx); 166533965Sjdp return (0); 166633965Sjdp } 166733965Sjdp } 166833965Sjdp mtx_unlock(&sc->sc_msi_mtx); 166933965Sjdp return (ENXIO); 167033965Sjdp} 167133965Sjdp 167233965Sjdpstatic int 167333965Sjdpfire_release_msi(device_t dev, device_t child, int count, int *irqs) 167433965Sjdp{ 167533965Sjdp struct fire_softc *sc; 167633965Sjdp u_int i; 167733965Sjdp 167833965Sjdp sc = device_get_softc(dev); 167933965Sjdp mtx_lock(&sc->sc_msi_mtx); 168033965Sjdp for (i = 0; i < count; i++) { 168133965Sjdp clrbit(sc->sc_msiq_bitmap, 168233965Sjdp sc->sc_msi_msiq_table[irqs[i] - sc->sc_msi_first]); 168333965Sjdp clrbit(sc->sc_msi_bitmap, irqs[i] - sc->sc_msi_first); 168433965Sjdp } 168533965Sjdp mtx_unlock(&sc->sc_msi_mtx); 168633965Sjdp return (0); 168733965Sjdp} 168833965Sjdp 168933965Sjdpstatic int 169033965Sjdpfire_alloc_msix(device_t dev, device_t child, int *irq) 169133965Sjdp{ 1692130561Sobrien struct fire_softc *sc; 1693130561Sobrien int i, msiq; 169433965Sjdp 169533965Sjdp sc = device_get_softc(dev); 169633965Sjdp if ((sc->sc_flags & FIRE_MSIX) == 0) 1697218822Sdim return (ENXIO); 1698218822Sdim mtx_lock(&sc->sc_msi_mtx); 1699130561Sobrien msiq = 0; 170033965Sjdp for (i = 0; i < sc->sc_msiq_count; i++) { 170133965Sjdp if (isclr(sc->sc_msiq_bitmap, i) != 0) { 1702218822Sdim msiq = i; 1703130561Sobrien break; 170433965Sjdp } 170533965Sjdp } 170660484Sobrien if (i == sc->sc_msiq_count) { 170733965Sjdp mtx_unlock(&sc->sc_msi_mtx); 170833965Sjdp return (ENXIO); 170933965Sjdp } 171033965Sjdp for (i = sc->sc_msi_count - 1; i >= 0; i--) { 171133965Sjdp if (isclr(sc->sc_msi_bitmap, i) != 0) { 171233965Sjdp setbit(sc->sc_msiq_bitmap, msiq); 171333965Sjdp setbit(sc->sc_msi_bitmap, i); 171433965Sjdp sc->sc_msi_msiq_table[i] = msiq; 171533965Sjdp *irq = sc->sc_msi_first + i; 171633965Sjdp mtx_unlock(&sc->sc_msi_mtx); 171733965Sjdp return (0); 171833965Sjdp } 171933965Sjdp } 172033965Sjdp mtx_unlock(&sc->sc_msi_mtx); 172133965Sjdp return (ENXIO); 172233965Sjdp} 172333965Sjdp 172433965Sjdpstatic int 172533965Sjdpfire_release_msix(device_t dev, device_t child, int irq) 172633965Sjdp{ 172733965Sjdp struct fire_softc *sc; 172833965Sjdp 172933965Sjdp sc = device_get_softc(dev); 173033965Sjdp if ((sc->sc_flags & FIRE_MSIX) == 0) 173133965Sjdp return (ENXIO); 173233965Sjdp mtx_lock(&sc->sc_msi_mtx); 173333965Sjdp clrbit(sc->sc_msiq_bitmap, 173433965Sjdp sc->sc_msi_msiq_table[irq - sc->sc_msi_first]); 173533965Sjdp clrbit(sc->sc_msi_bitmap, irq - sc->sc_msi_first); 1736130561Sobrien mtx_unlock(&sc->sc_msi_mtx); 173733965Sjdp return (0); 1738130561Sobrien} 173933965Sjdp 174033965Sjdpstatic int 1741130561Sobrienfire_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, 174233965Sjdp uint32_t *data) 174333965Sjdp{ 174433965Sjdp struct fire_softc *sc; 174533965Sjdp struct pci_devinfo *dinfo; 1746130561Sobrien 174733965Sjdp sc = device_get_softc(dev); 174833965Sjdp dinfo = device_get_ivars(child); 174933965Sjdp if (dinfo->cfg.msi.msi_alloc > 0) { 175033965Sjdp if ((irq & ~sc->sc_msi_data_mask) != 0) { 175133965Sjdp device_printf(dev, "invalid MSI 0x%x\n", irq); 175233965Sjdp return (EINVAL); 1753130561Sobrien } 175433965Sjdp } else { 1755130561Sobrien if ((sc->sc_flags & FIRE_MSIX) == 0) 175633965Sjdp return (ENXIO); 1757130561Sobrien if (fls(irq) > sc->sc_msix_data_width) { 175860484Sobrien device_printf(dev, "invalid MSI-X 0x%x\n", irq); 1759130561Sobrien return (EINVAL); 176060484Sobrien } 176177298Sobrien } 1762130561Sobrien if (dinfo->cfg.msi.msi_alloc > 0 && 176360484Sobrien (dinfo->cfg.msi.msi_ctrl & PCIM_MSICTRL_64BIT) == 0) 176433965Sjdp *addr = sc->sc_msi_addr32; 176533965Sjdp else 176633965Sjdp *addr = sc->sc_msi_addr64; 176733965Sjdp *data = irq; 1768130561Sobrien return (0); 176960484Sobrien} 177033965Sjdp 177133965Sjdpstatic void 1772130561Sobrienfire_msiq_handler(void *cookie) 177333965Sjdp{ 1774130561Sobrien struct intr_vector *iv; 177533965Sjdp struct fire_msiqarg *fmqa; 1776130561Sobrien 177760484Sobrien iv = cookie; 177860484Sobrien fmqa = iv->iv_icarg; 1779130561Sobrien /* 178060484Sobrien * Note that since fire_intr_clear() will clear the event queue 178133965Sjdp * interrupt after the handler associated with the MSI [sic] has 178233965Sjdp * been executed we have to protect the access to the event queue as 178333965Sjdp * otherwise nested event queue interrupts cause corruption of the 178433965Sjdp * event queue on MP machines. Obviously especially when abandoning 178533965Sjdp * the 1:1 mapping it would be better to not clear the event queue 178633965Sjdp * interrupt after each handler invocation but only once when the 178733965Sjdp * outstanding MSIs have been processed but unfortunately that 178833965Sjdp * doesn't work well and leads to interrupt storms with controllers/ 178933965Sjdp * drivers which don't mask interrupts while the handler is executed. 179033965Sjdp * Maybe delaying clearing the MSI until after the handler has been 179133965Sjdp * executed could be used to work around this but that's not the 179233965Sjdp * intended usage and might in turn cause lost MSIs. 179333965Sjdp */ 179433965Sjdp mtx_lock_spin(&fmqa->fmqa_mtx); 179533965Sjdp fire_msiq_common(iv, fmqa); 179633965Sjdp mtx_unlock_spin(&fmqa->fmqa_mtx); 179733965Sjdp} 179860484Sobrien 1799130561Sobrienstatic void 180033965Sjdpfire_msiq_filter(void *cookie) 180133965Sjdp{ 180233965Sjdp struct intr_vector *iv; 180333965Sjdp struct fire_msiqarg *fmqa; 180433965Sjdp 180533965Sjdp iv = cookie; 180633965Sjdp fmqa = iv->iv_icarg; 180733965Sjdp /* 180833965Sjdp * For filters we don't use fire_intr_clear() since it would clear 180933965Sjdp * the event queue interrupt while we're still processing the event 181033965Sjdp * queue as filters and associated post-filter handler are executed 181133965Sjdp * directly, which in turn would lead to lost MSIs. So we clear the 181233965Sjdp * event queue interrupt only once after processing the event queue. 181333965Sjdp * Given that this still guarantees the filters to not be executed 181433965Sjdp * concurrently and no other CPU can clear the event queue interrupt 181533965Sjdp * while the event queue is still processed, we don't even need to 181633965Sjdp * interlock the access to the event queue in this case. 181733965Sjdp */ 1818130561Sobrien critical_enter(); 181933965Sjdp fire_msiq_common(iv, fmqa); 182033965Sjdp FIRE_PCI_WRITE_8(fmqa->fmqa_fica.fica_sc, fmqa->fmqa_fica.fica_clr, 1821130561Sobrien INTCLR_IDLE); 182233965Sjdp critical_exit(); 182333965Sjdp} 182433965Sjdp 182533965Sjdpstatic inline void 182633965Sjdpfire_msiq_common(struct intr_vector *iv, struct fire_msiqarg *fmqa) 182733965Sjdp{ 182833965Sjdp struct fire_softc *sc; 1829130561Sobrien struct fo_msiq_record *qrec; 183033965Sjdp device_t dev; 183133965Sjdp uint64_t word0; 1832130561Sobrien u_int head, msi, msiq; 183333965Sjdp 183433965Sjdp sc = fmqa->fmqa_fica.fica_sc; 183533965Sjdp dev = sc->sc_dev; 183633965Sjdp msiq = fmqa->fmqa_msiq; 183733965Sjdp head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >> 183833965Sjdp FO_PCI_EQ_HD_SHFT; 183933965Sjdp qrec = &fmqa->fmqa_base[head]; 184033965Sjdp word0 = qrec->fomqr_word0; 184133965Sjdp for (;;) { 184233965Sjdp if (__predict_false((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0)) 184333965Sjdp break; 184433965Sjdp KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSI64) != 0 || 184533965Sjdp (word0 & FO_MQR_WORD0_FMT_TYPE_MSI32) != 0, 184633965Sjdp ("%s: received non-MSI/MSI-X message in event queue %d " 184733965Sjdp "(word0 %#llx)", device_get_nameunit(dev), msiq, 184833965Sjdp (unsigned long long)word0)); 184933965Sjdp msi = (word0 & FO_MQR_WORD0_DATA0_MASK) >> 185033965Sjdp FO_MQR_WORD0_DATA0_SHFT; 1851130561Sobrien /* 185233965Sjdp * Sanity check the MSI/MSI-X as long as we use a 1:1 mapping. 185333965Sjdp */ 185433965Sjdp KASSERT(msi == fmqa->fmqa_msi, 185533965Sjdp ("%s: received non-matching MSI/MSI-X in event queue %d " 185633965Sjdp "(%d versus %d)", device_get_nameunit(dev), msiq, msi, 185733965Sjdp fmqa->fmqa_msi)); 185833965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + (msi << 3), 1859104834Sobrien FO_PCI_MSI_CLR_EQWR_N); 1860130561Sobrien if (__predict_false(intr_event_handle(iv->iv_event, 186133965Sjdp NULL) != 0)) 186233965Sjdp printf("stray MSI/MSI-X in event queue %d\n", msiq); 186333965Sjdp qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK; 186433965Sjdp head = (head + 1) % sc->sc_msiq_size; 186533965Sjdp qrec = &fmqa->fmqa_base[head]; 186633965Sjdp word0 = qrec->fomqr_word0; 186733965Sjdp } 1868104834Sobrien FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) << 1869130561Sobrien FO_PCI_EQ_HD_SHFT); 187033965Sjdp if (__predict_false((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) & 187133965Sjdp FO_PCI_EQ_TL_OVERR) != 0)) { 187233965Sjdp device_printf(dev, "event queue %d overflow\n", msiq); 187333965Sjdp msiq <<= 3; 187433965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, 187533965Sjdp FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) | 187633965Sjdp FO_PCI_EQ_CTRL_CLR_COVERR); 187733965Sjdp } 187833965Sjdp} 187933965Sjdp 188033965Sjdpstatic int 188133965Sjdpfire_setup_intr(device_t dev, device_t child, struct resource *ires, 188233965Sjdp int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, 188333965Sjdp void **cookiep) 188433965Sjdp{ 188533965Sjdp struct fire_softc *sc; 188633965Sjdp struct fire_msiqarg *fmqa; 188733965Sjdp u_long vec; 188833965Sjdp int error; 1889130561Sobrien u_int msi, msiq; 189033965Sjdp 189133965Sjdp sc = device_get_softc(dev); 189233965Sjdp /* 189333965Sjdp * XXX this assumes that a device only has one INTx, while in fact 189433965Sjdp * Cassini+ and Saturn can use all four the firmware has assigned 189533965Sjdp * to them, but so does pci(4). 189633965Sjdp */ 189733965Sjdp if (rman_get_rid(ires) != 0) { 189833965Sjdp msi = rman_get_start(ires); 1899130561Sobrien msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first]; 190033965Sjdp vec = INTMAP_VEC(sc->sc_ign, sc->sc_msiq_ino_first + msiq); 190133965Sjdp msiq += sc->sc_msiq_first; 190233965Sjdp if (intr_vectors[vec].iv_ic != &fire_ic) { 190333965Sjdp device_printf(dev, 190433965Sjdp "invalid interrupt controller for vector 0x%lx\n", 190533965Sjdp vec); 190633965Sjdp return (EINVAL); 1907130561Sobrien } 190833965Sjdp /* 190933965Sjdp * The MD interrupt code needs the vector rather than the MSI. 191033965Sjdp */ 191133965Sjdp rman_set_start(ires, vec); 191233965Sjdp rman_set_end(ires, vec); 191333965Sjdp error = bus_generic_setup_intr(dev, child, ires, flags, filt, 191433965Sjdp intr, arg, cookiep); 191533965Sjdp rman_set_start(ires, msi); 191633965Sjdp rman_set_end(ires, msi); 191733965Sjdp if (error != 0) 191833965Sjdp return (error); 1919104834Sobrien fmqa = intr_vectors[vec].iv_icarg; 192033965Sjdp /* 192133965Sjdp * XXX inject our event queue handler. 192233965Sjdp */ 192333965Sjdp if (filt != NULL) { 192433965Sjdp intr_vectors[vec].iv_func = fire_msiq_filter; 1925130561Sobrien intr_vectors[vec].iv_ic = &fire_msiqc_filter; 192633965Sjdp /* 192733965Sjdp * Ensure the event queue interrupt is cleared, it 192833965Sjdp * might have triggered before. Given we supply NULL 192933965Sjdp * as ic_clear, inthand_add() won't do this for us. 193033965Sjdp */ 193133965Sjdp FIRE_PCI_WRITE_8(sc, fmqa->fmqa_fica.fica_clr, 193233965Sjdp INTCLR_IDLE); 193333965Sjdp } else 193433965Sjdp intr_vectors[vec].iv_func = fire_msiq_handler; 193533965Sjdp /* Record the MSI/MSI-X as long as we we use a 1:1 mapping. */ 193633965Sjdp fmqa->fmqa_msi = msi; 193733965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (msiq << 3), 193833965Sjdp FO_PCI_EQ_CTRL_SET_EN); 193933965Sjdp msi <<= 3; 194033965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, 194133965Sjdp (FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) & 194233965Sjdp ~FO_PCI_MSI_MAP_EQNUM_MASK) | 194333965Sjdp ((msiq << FO_PCI_MSI_MAP_EQNUM_SHFT) & 194433965Sjdp FO_PCI_MSI_MAP_EQNUM_MASK)); 194533965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + msi, 194633965Sjdp FO_PCI_MSI_CLR_EQWR_N); 194733965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, 194833965Sjdp FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) | 194933965Sjdp FO_PCI_MSI_MAP_V); 195033965Sjdp return (error); 195133965Sjdp } 195233965Sjdp 195333965Sjdp /* 195433965Sjdp * Make sure the vector is fully specified and we registered 195533965Sjdp * our interrupt controller for it. 195633965Sjdp */ 195733965Sjdp vec = rman_get_start(ires); 195833965Sjdp if (INTIGN(vec) != sc->sc_ign) { 195933965Sjdp device_printf(dev, "invalid interrupt vector 0x%lx\n", vec); 196033965Sjdp return (EINVAL); 196133965Sjdp } 196233965Sjdp if (intr_vectors[vec].iv_ic != &fire_ic) { 196333965Sjdp device_printf(dev, 196433965Sjdp "invalid interrupt controller for vector 0x%lx\n", vec); 196533965Sjdp return (EINVAL); 1966130561Sobrien } 196733965Sjdp return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr, 196833965Sjdp arg, cookiep)); 196933965Sjdp} 197033965Sjdp 197133965Sjdpstatic int 197233965Sjdpfire_teardown_intr(device_t dev, device_t child, struct resource *ires, 197333965Sjdp void *cookie) 197433965Sjdp{ 197533965Sjdp struct fire_softc *sc; 197633965Sjdp u_long vec; 197733965Sjdp int error; 1978130561Sobrien u_int msi, msiq; 197933965Sjdp 1980130561Sobrien sc = device_get_softc(dev); 198133965Sjdp if (rman_get_rid(ires) != 0) { 198233965Sjdp msi = rman_get_start(ires); 198333965Sjdp msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first]; 198433965Sjdp vec = INTMAP_VEC(sc->sc_ign, msiq + sc->sc_msiq_ino_first); 198533965Sjdp msiq += sc->sc_msiq_first; 198633965Sjdp msi <<= 3; 198733965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, 198833965Sjdp FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) & 198933965Sjdp ~FO_PCI_MSI_MAP_V); 199033965Sjdp msiq <<= 3; 199133965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, 199233965Sjdp FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I | 199333965Sjdp FO_PCI_EQ_CTRL_CLR_DIS); 199433965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_TL_BASE + msiq, 199533965Sjdp (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK); 199633965Sjdp FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_HD_BASE + msiq, 199733965Sjdp (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK); 199833965Sjdp intr_vectors[vec].iv_ic = &fire_ic; 199933965Sjdp /* 200033965Sjdp * The MD interrupt code needs the vector rather than the MSI. 2001130561Sobrien */ 200233965Sjdp rman_set_start(ires, vec); 200333965Sjdp rman_set_end(ires, vec); 200433965Sjdp error = bus_generic_teardown_intr(dev, child, ires, cookie); 2005130561Sobrien msi >>= 3; 2006130561Sobrien rman_set_start(ires, msi); 2007130561Sobrien rman_set_end(ires, msi); 200833965Sjdp return (error); 200933965Sjdp } 201033965Sjdp return (bus_generic_teardown_intr(dev, child, ires, cookie)); 201133965Sjdp} 201233965Sjdp 2013130561Sobrienstatic struct resource * 201433965Sjdpfire_alloc_resource(device_t bus, device_t child, int type, int *rid, 201533965Sjdp u_long start, u_long end, u_long count, u_int flags) 2016130561Sobrien{ 201733965Sjdp struct fire_softc *sc; 201833965Sjdp struct resource *rv; 201933965Sjdp struct rman *rm; 202033965Sjdp 2021130561Sobrien sc = device_get_softc(bus); 202233965Sjdp switch (type) { 202333965Sjdp case SYS_RES_IRQ: 202433965Sjdp /* 202533965Sjdp * XXX: Don't accept blank ranges for now, only single 202633965Sjdp * interrupts. The other case should not happen with 202733965Sjdp * the MI PCI code... 202833965Sjdp * XXX: This may return a resource that is out of the 202933965Sjdp * range that was specified. Is this correct...? 203033965Sjdp */ 203133965Sjdp if (start != end) 203233965Sjdp panic("%s: XXX: interrupt range", __func__); 203333965Sjdp if (*rid == 0) 203433965Sjdp start = end = INTMAP_VEC(sc->sc_ign, end); 203533965Sjdp return (bus_generic_alloc_resource(bus, child, type, rid, 203633965Sjdp start, end, count, flags)); 203733965Sjdp case SYS_RES_MEMORY: 203833965Sjdp rm = &sc->sc_pci_mem_rman; 203933965Sjdp break; 204033965Sjdp case SYS_RES_IOPORT: 204133965Sjdp rm = &sc->sc_pci_io_rman; 204233965Sjdp break; 204333965Sjdp default: 204433965Sjdp return (NULL); 204533965Sjdp } 204633965Sjdp 204733965Sjdp rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE, 204833965Sjdp child); 204933965Sjdp if (rv == NULL) 205033965Sjdp return (NULL); 205133965Sjdp rman_set_rid(rv, *rid); 205233965Sjdp 205333965Sjdp if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type, 2054130561Sobrien *rid, rv) != 0) { 205533965Sjdp rman_release_resource(rv); 205633965Sjdp return (NULL); 205733965Sjdp } 205833965Sjdp return (rv); 205933965Sjdp} 206033965Sjdp 206133965Sjdpstatic int 206233965Sjdpfire_activate_resource(device_t bus, device_t child, int type, int rid, 206333965Sjdp struct resource *r) 2064130561Sobrien{ 206533965Sjdp struct fire_softc *sc; 2066130561Sobrien struct bus_space_tag *tag; 2067130561Sobrien 2068130561Sobrien sc = device_get_softc(bus); 206933965Sjdp switch (type) { 207033965Sjdp case SYS_RES_IRQ: 207133965Sjdp return (bus_generic_activate_resource(bus, child, type, rid, 207233965Sjdp r)); 207333965Sjdp case SYS_RES_MEMORY: 207433965Sjdp tag = sparc64_alloc_bus_tag(r, rman_get_bustag( 207533965Sjdp sc->sc_mem_res[FIRE_PCI]), PCI_MEMORY_BUS_SPACE, NULL); 207633965Sjdp if (tag == NULL) 207733965Sjdp return (ENOMEM); 207833965Sjdp rman_set_bustag(r, tag); 207933965Sjdp rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] + 208033965Sjdp rman_get_start(r)); 2081130561Sobrien break; 208233965Sjdp case SYS_RES_IOPORT: 208333965Sjdp rman_set_bustag(r, sc->sc_pci_iot); 208433965Sjdp rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] + 2085130561Sobrien rman_get_start(r)); 208633965Sjdp break; 208733965Sjdp } 208833965Sjdp return (rman_activate_resource(r)); 208933965Sjdp} 2090130561Sobrien 209133965Sjdpstatic int 209233965Sjdpfire_adjust_resource(device_t bus, device_t child, int type, 209333965Sjdp struct resource *r, u_long start, u_long end) 209433965Sjdp{ 209533965Sjdp struct fire_softc *sc; 209633965Sjdp struct rman *rm; 209733965Sjdp 2098130561Sobrien sc = device_get_softc(bus); 209933965Sjdp switch (type) { 210033965Sjdp case SYS_RES_IRQ: 210133965Sjdp return (bus_generic_adjust_resource(bus, child, type, r, 210233965Sjdp start, end)); 210333965Sjdp case SYS_RES_MEMORY: 210433965Sjdp rm = &sc->sc_pci_mem_rman; 210533965Sjdp break; 2106130561Sobrien case SYS_RES_IOPORT: 210733965Sjdp rm = &sc->sc_pci_io_rman; 210833965Sjdp break; 2109130561Sobrien default: 211033965Sjdp return (EINVAL); 211133965Sjdp } 211260484Sobrien if (rman_is_region_manager(r, rm) == 0) 2113130561Sobrien return (EINVAL); 211433965Sjdp return (rman_adjust_resource(r, start, end)); 211533965Sjdp} 211633965Sjdp 211733965Sjdpstatic bus_dma_tag_t 211833965Sjdpfire_get_dma_tag(device_t bus, device_t child __unused) 211933965Sjdp{ 212033965Sjdp struct fire_softc *sc; 212133965Sjdp 212233965Sjdp sc = device_get_softc(bus); 212333965Sjdp return (sc->sc_pci_dmat); 212433965Sjdp} 212533965Sjdp 212633965Sjdpstatic phandle_t 212733965Sjdpfire_get_node(device_t bus, device_t child __unused) 212833965Sjdp{ 212933965Sjdp struct fire_softc *sc; 213060484Sobrien 213133965Sjdp sc = device_get_softc(bus); 213233965Sjdp /* We only have one child, the PCI bus, which needs our own node. */ 213333965Sjdp return (sc->sc_node); 213433965Sjdp} 213533965Sjdp 213633965Sjdpstatic u_int 213733965Sjdpfire_get_timecount(struct timecounter *tc) 213833965Sjdp{ 2139130561Sobrien struct fire_softc *sc; 214033965Sjdp 214133965Sjdp sc = tc->tc_priv; 214233965Sjdp return (FIRE_CTRL_READ_8(sc, FO_XBC_PRF_CNT0) & TC_COUNTER_MAX_MASK); 2143130561Sobrien} 214433965Sjdp