1195534Sscottl/*- 2238805Smav * Copyright (c) 2009-2012 Alexander Motin <mav@FreeBSD.org> 3195534Sscottl * All rights reserved. 4195534Sscottl * 5195534Sscottl * Redistribution and use in source and binary forms, with or without 6195534Sscottl * modification, are permitted provided that the following conditions 7195534Sscottl * are met: 8195534Sscottl * 1. Redistributions of source code must retain the above copyright 9195534Sscottl * notice, this list of conditions and the following disclaimer, 10195534Sscottl * without modification, immediately at the beginning of the file. 11195534Sscottl * 2. Redistributions in binary form must reproduce the above copyright 12195534Sscottl * notice, this list of conditions and the following disclaimer in the 13195534Sscottl * documentation and/or other materials provided with the distribution. 14195534Sscottl * 15195534Sscottl * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16195534Sscottl * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17195534Sscottl * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18195534Sscottl * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19195534Sscottl * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20195534Sscottl * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21195534Sscottl * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22195534Sscottl * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23195534Sscottl * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24195534Sscottl * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25195534Sscottl */ 26195534Sscottl 27195534Sscottl#include <sys/cdefs.h> 28195534Sscottl__FBSDID("$FreeBSD: stable/10/sys/dev/ahci/ahci.c 315813 2017-03-23 06:41:13Z mav $"); 29195534Sscottl 30195534Sscottl#include <sys/param.h> 31195534Sscottl#include <sys/module.h> 32195534Sscottl#include <sys/systm.h> 33195534Sscottl#include <sys/kernel.h> 34195534Sscottl#include <sys/bus.h> 35220576Smav#include <sys/conf.h> 36195534Sscottl#include <sys/endian.h> 37195534Sscottl#include <sys/malloc.h> 38195534Sscottl#include <sys/lock.h> 39195534Sscottl#include <sys/mutex.h> 40195534Sscottl#include <machine/stdarg.h> 41195534Sscottl#include <machine/resource.h> 42195534Sscottl#include <machine/bus.h> 43195534Sscottl#include <sys/rman.h> 44195534Sscottl#include "ahci.h" 45195534Sscottl 46195534Sscottl#include <cam/cam.h> 47195534Sscottl#include <cam/cam_ccb.h> 48195534Sscottl#include <cam/cam_sim.h> 49195534Sscottl#include <cam/cam_xpt_sim.h> 50195534Sscottl#include <cam/cam_debug.h> 51195534Sscottl 52195534Sscottl/* local prototypes */ 53195534Sscottlstatic void ahci_intr(void *data); 54195534Sscottlstatic void ahci_intr_one(void *data); 55260387Sscottlstatic void ahci_intr_one_edge(void *data); 56208375Smavstatic int ahci_ch_init(device_t dev); 57208375Smavstatic int ahci_ch_deinit(device_t dev); 58195534Sscottlstatic int ahci_ch_suspend(device_t dev); 59195534Sscottlstatic int ahci_ch_resume(device_t dev); 60196656Smavstatic void ahci_ch_pm(void *arg); 61260387Sscottlstatic void ahci_ch_intr(void *arg); 62260387Sscottlstatic void ahci_ch_intr_direct(void *arg); 63260387Sscottlstatic void ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus); 64279918Smavstatic void ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb); 65195534Sscottlstatic void ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error); 66195534Sscottlstatic void ahci_execute_transaction(struct ahci_slot *slot); 67195534Sscottlstatic void ahci_timeout(struct ahci_slot *slot); 68195534Sscottlstatic void ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et); 69279918Smavstatic int ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag); 70195534Sscottlstatic void ahci_dmainit(device_t dev); 71195534Sscottlstatic void ahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); 72195534Sscottlstatic void ahci_dmafini(device_t dev); 73195534Sscottlstatic void ahci_slotsalloc(device_t dev); 74195534Sscottlstatic void ahci_slotsfree(device_t dev); 75279918Smavstatic void ahci_reset(struct ahci_channel *ch); 76279918Smavstatic void ahci_start(struct ahci_channel *ch, int fbs); 77279918Smavstatic void ahci_stop(struct ahci_channel *ch); 78279918Smavstatic void ahci_clo(struct ahci_channel *ch); 79279918Smavstatic void ahci_start_fr(struct ahci_channel *ch); 80279918Smavstatic void ahci_stop_fr(struct ahci_channel *ch); 81195534Sscottl 82195534Sscottlstatic int ahci_sata_connect(struct ahci_channel *ch); 83279918Smavstatic int ahci_sata_phy_reset(struct ahci_channel *ch); 84279918Smavstatic int ahci_wait_ready(struct ahci_channel *ch, int t, int t0); 85195534Sscottl 86279918Smavstatic void ahci_issue_recovery(struct ahci_channel *ch); 87279918Smavstatic void ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb); 88279918Smavstatic void ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb); 89195534Sscottl 90195534Sscottlstatic void ahciaction(struct cam_sim *sim, union ccb *ccb); 91195534Sscottlstatic void ahcipoll(struct cam_sim *sim); 92195534Sscottl 93227293Sedstatic MALLOC_DEFINE(M_AHCI, "AHCI driver", "AHCI driver data buffers"); 94195534Sscottl 95220565Smav#define recovery_type spriv_field0 96220565Smav#define RECOVERY_NONE 0 97220565Smav#define RECOVERY_READ_LOG 1 98220565Smav#define RECOVERY_REQUEST_SENSE 2 99220565Smav#define recovery_slot spriv_field1 100220565Smav 101279918Smavint 102279918Smavahci_ctlr_setup(device_t dev) 103195534Sscottl{ 104279918Smav struct ahci_controller *ctlr = device_get_softc(dev); 105279918Smav /* Clear interrupts */ 106279918Smav ATA_OUTL(ctlr->r_mem, AHCI_IS, ATA_INL(ctlr->r_mem, AHCI_IS)); 107279918Smav /* Configure CCC */ 108279918Smav if (ctlr->ccc) { 109279918Smav ATA_OUTL(ctlr->r_mem, AHCI_CCCP, ATA_INL(ctlr->r_mem, AHCI_PI)); 110279918Smav ATA_OUTL(ctlr->r_mem, AHCI_CCCC, 111279918Smav (ctlr->ccc << AHCI_CCCC_TV_SHIFT) | 112279918Smav (4 << AHCI_CCCC_CC_SHIFT) | 113279918Smav AHCI_CCCC_EN); 114279918Smav ctlr->cccv = (ATA_INL(ctlr->r_mem, AHCI_CCCC) & 115279918Smav AHCI_CCCC_INT_MASK) >> AHCI_CCCC_INT_SHIFT; 116279918Smav if (bootverbose) { 117279918Smav device_printf(dev, 118279918Smav "CCC with %dms/4cmd enabled on vector %d\n", 119279918Smav ctlr->ccc, ctlr->cccv); 120199322Smav } 121199322Smav } 122279918Smav /* Enable AHCI interrupts */ 123279918Smav ATA_OUTL(ctlr->r_mem, AHCI_GHC, 124279918Smav ATA_INL(ctlr->r_mem, AHCI_GHC) | AHCI_GHC_IE); 125279918Smav return (0); 126199322Smav} 127199322Smav 128279918Smavint 129279918Smavahci_ctlr_reset(device_t dev) 130199322Smav{ 131279918Smav struct ahci_controller *ctlr = device_get_softc(dev); 132279918Smav int timeout; 133195534Sscottl 134279918Smav /* Enable AHCI mode */ 135279918Smav ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE); 136279918Smav /* Reset AHCI controller */ 137279918Smav ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE|AHCI_GHC_HR); 138279918Smav for (timeout = 1000; timeout > 0; timeout--) { 139279918Smav DELAY(1000); 140279918Smav if ((ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_HR) == 0) 141279918Smav break; 142279918Smav } 143279918Smav if (timeout == 0) { 144279918Smav device_printf(dev, "AHCI controller reset failure\n"); 145199322Smav return (ENXIO); 146199176Smav } 147279918Smav /* Reenable AHCI mode */ 148279918Smav ATA_OUTL(ctlr->r_mem, AHCI_GHC, AHCI_GHC_AE); 149279918Smav return (0); 150195534Sscottl} 151195534Sscottl 152279918Smav 153279918Smavint 154195534Sscottlahci_attach(device_t dev) 155195534Sscottl{ 156195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 157302791Smav int error, i, speed, unit; 158302791Smav uint32_t u, version; 159195534Sscottl device_t child; 160195534Sscottl 161195534Sscottl ctlr->dev = dev; 162279918Smav ctlr->ccc = 0; 163196656Smav resource_int_value(device_get_name(dev), 164196656Smav device_get_unit(dev), "ccc", &ctlr->ccc); 165279918Smav 166195534Sscottl /* Setup our own memory management for channels. */ 167208414Smav ctlr->sc_iomem.rm_start = rman_get_start(ctlr->r_mem); 168208414Smav ctlr->sc_iomem.rm_end = rman_get_end(ctlr->r_mem); 169195534Sscottl ctlr->sc_iomem.rm_type = RMAN_ARRAY; 170195534Sscottl ctlr->sc_iomem.rm_descr = "I/O memory addresses"; 171195534Sscottl if ((error = rman_init(&ctlr->sc_iomem)) != 0) { 172195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 173195534Sscottl return (error); 174195534Sscottl } 175195534Sscottl if ((error = rman_manage_region(&ctlr->sc_iomem, 176195534Sscottl rman_get_start(ctlr->r_mem), rman_get_end(ctlr->r_mem))) != 0) { 177195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 178195534Sscottl rman_fini(&ctlr->sc_iomem); 179195534Sscottl return (error); 180195534Sscottl } 181199322Smav /* Get the HW capabilities */ 182199322Smav version = ATA_INL(ctlr->r_mem, AHCI_VS); 183199322Smav ctlr->caps = ATA_INL(ctlr->r_mem, AHCI_CAP); 184240383Smav if (version >= 0x00010200) 185199322Smav ctlr->caps2 = ATA_INL(ctlr->r_mem, AHCI_CAP2); 186203108Smav if (ctlr->caps & AHCI_CAP_EMS) 187203108Smav ctlr->capsem = ATA_INL(ctlr->r_mem, AHCI_EM_CTL); 188195534Sscottl ctlr->ichannels = ATA_INL(ctlr->r_mem, AHCI_PI); 189222304Smav 190222304Smav /* Identify and set separate quirks for HBA and RAID f/w Marvells. */ 191271523Smav if ((ctlr->quirks & AHCI_Q_ALTSIG) && 192222304Smav (ctlr->caps & AHCI_CAP_SPM) == 0) 193271523Smav ctlr->quirks |= AHCI_Q_NOBSYRES; 194222304Smav 195199322Smav if (ctlr->quirks & AHCI_Q_1CH) { 196199322Smav ctlr->caps &= ~AHCI_CAP_NPMASK; 197199322Smav ctlr->ichannels &= 0x01; 198199322Smav } 199199322Smav if (ctlr->quirks & AHCI_Q_2CH) { 200199322Smav ctlr->caps &= ~AHCI_CAP_NPMASK; 201199322Smav ctlr->caps |= 1; 202199322Smav ctlr->ichannels &= 0x03; 203199322Smav } 204199322Smav if (ctlr->quirks & AHCI_Q_4CH) { 205199322Smav ctlr->caps &= ~AHCI_CAP_NPMASK; 206199322Smav ctlr->caps |= 3; 207199322Smav ctlr->ichannels &= 0x0f; 208199322Smav } 209195534Sscottl ctlr->channels = MAX(flsl(ctlr->ichannels), 210199322Smav (ctlr->caps & AHCI_CAP_NPMASK) + 1); 211199322Smav if (ctlr->quirks & AHCI_Q_NOPMP) 212199322Smav ctlr->caps &= ~AHCI_CAP_SPM; 213199322Smav if (ctlr->quirks & AHCI_Q_NONCQ) 214199322Smav ctlr->caps &= ~AHCI_CAP_SNCQ; 215205422Smav if ((ctlr->caps & AHCI_CAP_CCCS) == 0) 216205422Smav ctlr->ccc = 0; 217222039Smav ctlr->emloc = ATA_INL(ctlr->r_mem, AHCI_EM_LOC); 218249346Smav 219249346Smav /* Create controller-wide DMA tag. */ 220276268Sian if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, 221249346Smav (ctlr->caps & AHCI_CAP_64BIT) ? BUS_SPACE_MAXADDR : 222249346Smav BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, 223249346Smav BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, 224249346Smav 0, NULL, NULL, &ctlr->dma_tag)) { 225249346Smav bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, 226249346Smav ctlr->r_mem); 227249346Smav rman_fini(&ctlr->sc_iomem); 228277061Ssmh return (ENXIO); 229249346Smav } 230249346Smav 231205422Smav ahci_ctlr_setup(dev); 232279918Smav 233195534Sscottl /* Setup interrupts. */ 234277061Ssmh if ((error = ahci_setup_interrupt(dev)) != 0) { 235249346Smav bus_dma_tag_destroy(ctlr->dma_tag); 236279918Smav bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, 237279918Smav ctlr->r_mem); 238195534Sscottl rman_fini(&ctlr->sc_iomem); 239277061Ssmh return (error); 240195534Sscottl } 241279918Smav 242260387Sscottl i = 0; 243260387Sscottl for (u = ctlr->ichannels; u != 0; u >>= 1) 244260387Sscottl i += (u & 1); 245260387Sscottl ctlr->direct = (ctlr->msi && (ctlr->numirqs > 1 || i <= 3)); 246260387Sscottl resource_int_value(device_get_name(dev), device_get_unit(dev), 247260387Sscottl "direct", &ctlr->direct); 248195534Sscottl /* Announce HW capabilities. */ 249196656Smav speed = (ctlr->caps & AHCI_CAP_ISS) >> AHCI_CAP_ISS_SHIFT; 250195534Sscottl device_printf(dev, 251203123Smav "AHCI v%x.%02x with %d %sGbps ports, Port Multiplier %s%s\n", 252195534Sscottl ((version >> 20) & 0xf0) + ((version >> 16) & 0x0f), 253195534Sscottl ((version >> 4) & 0xf0) + (version & 0x0f), 254196656Smav (ctlr->caps & AHCI_CAP_NPMASK) + 1, 255195534Sscottl ((speed == 1) ? "1.5":((speed == 2) ? "3": 256195534Sscottl ((speed == 3) ? "6":"?"))), 257196656Smav (ctlr->caps & AHCI_CAP_SPM) ? 258203123Smav "supported" : "not supported", 259203123Smav (ctlr->caps & AHCI_CAP_FBSS) ? 260203123Smav " with FBS" : ""); 261250792Ssmh if (ctlr->quirks != 0) { 262250792Ssmh device_printf(dev, "quirks=0x%b\n", ctlr->quirks, 263250792Ssmh AHCI_Q_BIT_STRING); 264250792Ssmh } 265195534Sscottl if (bootverbose) { 266195534Sscottl device_printf(dev, "Caps:%s%s%s%s%s%s%s%s %sGbps", 267196656Smav (ctlr->caps & AHCI_CAP_64BIT) ? " 64bit":"", 268196656Smav (ctlr->caps & AHCI_CAP_SNCQ) ? " NCQ":"", 269196656Smav (ctlr->caps & AHCI_CAP_SSNTF) ? " SNTF":"", 270196656Smav (ctlr->caps & AHCI_CAP_SMPS) ? " MPS":"", 271196656Smav (ctlr->caps & AHCI_CAP_SSS) ? " SS":"", 272196656Smav (ctlr->caps & AHCI_CAP_SALP) ? " ALP":"", 273196656Smav (ctlr->caps & AHCI_CAP_SAL) ? " AL":"", 274196656Smav (ctlr->caps & AHCI_CAP_SCLO) ? " CLO":"", 275195534Sscottl ((speed == 1) ? "1.5":((speed == 2) ? "3": 276195534Sscottl ((speed == 3) ? "6":"?")))); 277195534Sscottl printf("%s%s%s%s%s%s %dcmd%s%s%s %dports\n", 278196656Smav (ctlr->caps & AHCI_CAP_SAM) ? " AM":"", 279196656Smav (ctlr->caps & AHCI_CAP_SPM) ? " PM":"", 280196656Smav (ctlr->caps & AHCI_CAP_FBSS) ? " FBS":"", 281196656Smav (ctlr->caps & AHCI_CAP_PMD) ? " PMD":"", 282196656Smav (ctlr->caps & AHCI_CAP_SSC) ? " SSC":"", 283196656Smav (ctlr->caps & AHCI_CAP_PSC) ? " PSC":"", 284196656Smav ((ctlr->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1, 285196656Smav (ctlr->caps & AHCI_CAP_CCCS) ? " CCC":"", 286196656Smav (ctlr->caps & AHCI_CAP_EMS) ? " EM":"", 287196656Smav (ctlr->caps & AHCI_CAP_SXS) ? " eSATA":"", 288196656Smav (ctlr->caps & AHCI_CAP_NPMASK) + 1); 289195534Sscottl } 290240383Smav if (bootverbose && version >= 0x00010200) { 291253647Smav device_printf(dev, "Caps2:%s%s%s%s%s%s\n", 292253647Smav (ctlr->caps2 & AHCI_CAP2_DESO) ? " DESO":"", 293253647Smav (ctlr->caps2 & AHCI_CAP2_SADM) ? " SADM":"", 294253647Smav (ctlr->caps2 & AHCI_CAP2_SDS) ? " SDS":"", 295196656Smav (ctlr->caps2 & AHCI_CAP2_APST) ? " APST":"", 296196656Smav (ctlr->caps2 & AHCI_CAP2_NVMP) ? " NVMP":"", 297196656Smav (ctlr->caps2 & AHCI_CAP2_BOH) ? " BOH":""); 298196656Smav } 299195534Sscottl /* Attach all channels on this controller */ 300195534Sscottl for (unit = 0; unit < ctlr->channels; unit++) { 301195534Sscottl child = device_add_child(dev, "ahcich", -1); 302227635Smav if (child == NULL) { 303195534Sscottl device_printf(dev, "failed to add channel device\n"); 304227635Smav continue; 305227635Smav } 306227635Smav device_set_ivars(child, (void *)(intptr_t)unit); 307227635Smav if ((ctlr->ichannels & (1 << unit)) == 0) 308227635Smav device_disable(child); 309195534Sscottl } 310238805Smav if (ctlr->caps & AHCI_CAP_EMS) { 311238805Smav child = device_add_child(dev, "ahciem", -1); 312238805Smav if (child == NULL) 313238805Smav device_printf(dev, "failed to add enclosure device\n"); 314238805Smav else 315238805Smav device_set_ivars(child, (void *)(intptr_t)-1); 316238805Smav } 317195534Sscottl bus_generic_attach(dev); 318277061Ssmh return (0); 319195534Sscottl} 320195534Sscottl 321279918Smavint 322195534Sscottlahci_detach(device_t dev) 323195534Sscottl{ 324195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 325227701Shselasky int i; 326195534Sscottl 327195534Sscottl /* Detach & delete all children */ 328227849Shselasky device_delete_children(dev); 329227701Shselasky 330195534Sscottl /* Free interrupts. */ 331195534Sscottl for (i = 0; i < ctlr->numirqs; i++) { 332195534Sscottl if (ctlr->irqs[i].r_irq) { 333195534Sscottl bus_teardown_intr(dev, ctlr->irqs[i].r_irq, 334195534Sscottl ctlr->irqs[i].handle); 335195534Sscottl bus_release_resource(dev, SYS_RES_IRQ, 336195534Sscottl ctlr->irqs[i].r_irq_rid, ctlr->irqs[i].r_irq); 337195534Sscottl } 338195534Sscottl } 339249346Smav bus_dma_tag_destroy(ctlr->dma_tag); 340195534Sscottl /* Free memory. */ 341195534Sscottl rman_fini(&ctlr->sc_iomem); 342195534Sscottl if (ctlr->r_mem) 343195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem); 344195534Sscottl return (0); 345195534Sscottl} 346195534Sscottl 347279918Smavint 348195534Sscottlahci_setup_interrupt(device_t dev) 349195534Sscottl{ 350195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 351260387Sscottl int i; 352195534Sscottl 353195534Sscottl /* Check for single MSI vector fallback. */ 354195534Sscottl if (ctlr->numirqs > 1 && 355195534Sscottl (ATA_INL(ctlr->r_mem, AHCI_GHC) & AHCI_GHC_MRSM) != 0) { 356195534Sscottl device_printf(dev, "Falling back to one MSI\n"); 357195534Sscottl ctlr->numirqs = 1; 358195534Sscottl } 359277061Ssmh 360277061Ssmh /* Ensure we don't overrun irqs. */ 361277061Ssmh if (ctlr->numirqs > AHCI_MAX_IRQS) { 362277061Ssmh device_printf(dev, "Too many irqs %d > %d (clamping)\n", 363277061Ssmh ctlr->numirqs, AHCI_MAX_IRQS); 364277061Ssmh ctlr->numirqs = AHCI_MAX_IRQS; 365277061Ssmh } 366277061Ssmh 367195534Sscottl /* Allocate all IRQs. */ 368195534Sscottl for (i = 0; i < ctlr->numirqs; i++) { 369195534Sscottl ctlr->irqs[i].ctlr = ctlr; 370260387Sscottl ctlr->irqs[i].r_irq_rid = i + (ctlr->msi ? 1 : 0); 371279918Smav if (ctlr->channels == 1 && !ctlr->ccc && ctlr->msi) 372279918Smav ctlr->irqs[i].mode = AHCI_IRQ_MODE_ONE; 373279918Smav else if (ctlr->numirqs == 1 || i >= ctlr->channels || 374196656Smav (ctlr->ccc && i == ctlr->cccv)) 375195534Sscottl ctlr->irqs[i].mode = AHCI_IRQ_MODE_ALL; 376304413Smav else if (ctlr->channels > ctlr->numirqs && 377304413Smav i == ctlr->numirqs - 1) 378195534Sscottl ctlr->irqs[i].mode = AHCI_IRQ_MODE_AFTER; 379195534Sscottl else 380195534Sscottl ctlr->irqs[i].mode = AHCI_IRQ_MODE_ONE; 381195534Sscottl if (!(ctlr->irqs[i].r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, 382195534Sscottl &ctlr->irqs[i].r_irq_rid, RF_SHAREABLE | RF_ACTIVE))) { 383195534Sscottl device_printf(dev, "unable to map interrupt\n"); 384277061Ssmh return (ENXIO); 385195534Sscottl } 386195534Sscottl if ((bus_setup_intr(dev, ctlr->irqs[i].r_irq, ATA_INTR_FLAGS, NULL, 387260387Sscottl (ctlr->irqs[i].mode != AHCI_IRQ_MODE_ONE) ? ahci_intr : 388260387Sscottl ((ctlr->quirks & AHCI_Q_EDGEIS) ? ahci_intr_one_edge : 389260387Sscottl ahci_intr_one), 390195534Sscottl &ctlr->irqs[i], &ctlr->irqs[i].handle))) { 391195534Sscottl /* SOS XXX release r_irq */ 392195534Sscottl device_printf(dev, "unable to setup interrupt\n"); 393277061Ssmh return (ENXIO); 394195534Sscottl } 395202011Smav if (ctlr->numirqs > 1) { 396202011Smav bus_describe_intr(dev, ctlr->irqs[i].r_irq, 397202011Smav ctlr->irqs[i].handle, 398202011Smav ctlr->irqs[i].mode == AHCI_IRQ_MODE_ONE ? 399202011Smav "ch%d" : "%d", i); 400202011Smav } 401195534Sscottl } 402195534Sscottl return (0); 403195534Sscottl} 404195534Sscottl 405195534Sscottl/* 406195534Sscottl * Common case interrupt handler. 407195534Sscottl */ 408195534Sscottlstatic void 409195534Sscottlahci_intr(void *data) 410195534Sscottl{ 411195534Sscottl struct ahci_controller_irq *irq = data; 412195534Sscottl struct ahci_controller *ctlr = irq->ctlr; 413205422Smav u_int32_t is, ise = 0; 414195534Sscottl void *arg; 415195534Sscottl int unit; 416195534Sscottl 417196656Smav if (irq->mode == AHCI_IRQ_MODE_ALL) { 418195534Sscottl unit = 0; 419196656Smav if (ctlr->ccc) 420196656Smav is = ctlr->ichannels; 421196656Smav else 422196656Smav is = ATA_INL(ctlr->r_mem, AHCI_IS); 423196656Smav } else { /* AHCI_IRQ_MODE_AFTER */ 424195534Sscottl unit = irq->r_irq_rid - 1; 425196656Smav is = ATA_INL(ctlr->r_mem, AHCI_IS); 426304414Smav is &= (0xffffffff << unit); 427196656Smav } 428205422Smav /* CCC interrupt is edge triggered. */ 429205422Smav if (ctlr->ccc) 430205422Smav ise = 1 << ctlr->cccv; 431200814Smav /* Some controllers have edge triggered IS. */ 432200814Smav if (ctlr->quirks & AHCI_Q_EDGEIS) 433205422Smav ise |= is; 434205422Smav if (ise != 0) 435205422Smav ATA_OUTL(ctlr->r_mem, AHCI_IS, ise); 436195534Sscottl for (; unit < ctlr->channels; unit++) { 437195534Sscottl if ((is & (1 << unit)) != 0 && 438195534Sscottl (arg = ctlr->interrupt[unit].argument)) { 439199322Smav ctlr->interrupt[unit].function(arg); 440195534Sscottl } 441195534Sscottl } 442200814Smav /* AHCI declares level triggered IS. */ 443200814Smav if (!(ctlr->quirks & AHCI_Q_EDGEIS)) 444200814Smav ATA_OUTL(ctlr->r_mem, AHCI_IS, is); 445195534Sscottl} 446195534Sscottl 447195534Sscottl/* 448195534Sscottl * Simplified interrupt handler for multivector MSI mode. 449195534Sscottl */ 450195534Sscottlstatic void 451195534Sscottlahci_intr_one(void *data) 452195534Sscottl{ 453195534Sscottl struct ahci_controller_irq *irq = data; 454195534Sscottl struct ahci_controller *ctlr = irq->ctlr; 455195534Sscottl void *arg; 456195534Sscottl int unit; 457195534Sscottl 458195534Sscottl unit = irq->r_irq_rid - 1; 459195534Sscottl if ((arg = ctlr->interrupt[unit].argument)) 460195534Sscottl ctlr->interrupt[unit].function(arg); 461202011Smav /* AHCI declares level triggered IS. */ 462260387Sscottl ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit); 463195534Sscottl} 464195534Sscottl 465260387Sscottlstatic void 466260387Sscottlahci_intr_one_edge(void *data) 467260387Sscottl{ 468260387Sscottl struct ahci_controller_irq *irq = data; 469260387Sscottl struct ahci_controller *ctlr = irq->ctlr; 470260387Sscottl void *arg; 471260387Sscottl int unit; 472260387Sscottl 473260387Sscottl unit = irq->r_irq_rid - 1; 474260387Sscottl /* Some controllers have edge triggered IS. */ 475260387Sscottl ATA_OUTL(ctlr->r_mem, AHCI_IS, 1 << unit); 476260387Sscottl if ((arg = ctlr->interrupt[unit].argument)) 477260387Sscottl ctlr->interrupt[unit].function(arg); 478260387Sscottl} 479260387Sscottl 480279918Smavstruct resource * 481195534Sscottlahci_alloc_resource(device_t dev, device_t child, int type, int *rid, 482279918Smav u_long start, u_long end, u_long count, u_int flags) 483195534Sscottl{ 484195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 485238805Smav struct resource *res; 486195534Sscottl long st; 487238805Smav int offset, size, unit; 488195534Sscottl 489238805Smav unit = (intptr_t)device_get_ivars(child); 490238805Smav res = NULL; 491195534Sscottl switch (type) { 492195534Sscottl case SYS_RES_MEMORY: 493238805Smav if (unit >= 0) { 494238805Smav offset = AHCI_OFFSET + (unit << 7); 495238805Smav size = 128; 496238805Smav } else if (*rid == 0) { 497238805Smav offset = AHCI_EM_CTL; 498238805Smav size = 4; 499238805Smav } else { 500238805Smav offset = (ctlr->emloc & 0xffff0000) >> 14; 501238805Smav size = (ctlr->emloc & 0x0000ffff) << 2; 502238805Smav if (*rid != 1) { 503238805Smav if (*rid == 2 && (ctlr->capsem & 504238805Smav (AHCI_EM_XMT | AHCI_EM_SMB)) == 0) 505238805Smav offset += size; 506238805Smav else 507238805Smav break; 508238805Smav } 509238805Smav } 510195534Sscottl st = rman_get_start(ctlr->r_mem); 511195534Sscottl res = rman_reserve_resource(&ctlr->sc_iomem, st + offset, 512238805Smav st + offset + size - 1, size, RF_ACTIVE, child); 513195534Sscottl if (res) { 514195534Sscottl bus_space_handle_t bsh; 515195534Sscottl bus_space_tag_t bst; 516195534Sscottl bsh = rman_get_bushandle(ctlr->r_mem); 517195534Sscottl bst = rman_get_bustag(ctlr->r_mem); 518195534Sscottl bus_space_subregion(bst, bsh, offset, 128, &bsh); 519195534Sscottl rman_set_bushandle(res, bsh); 520195534Sscottl rman_set_bustag(res, bst); 521195534Sscottl } 522195534Sscottl break; 523195534Sscottl case SYS_RES_IRQ: 524195534Sscottl if (*rid == ATA_IRQ_RID) 525195534Sscottl res = ctlr->irqs[0].r_irq; 526195534Sscottl break; 527195534Sscottl } 528195534Sscottl return (res); 529195534Sscottl} 530195534Sscottl 531279918Smavint 532195534Sscottlahci_release_resource(device_t dev, device_t child, int type, int rid, 533279918Smav struct resource *r) 534195534Sscottl{ 535195534Sscottl 536195534Sscottl switch (type) { 537195534Sscottl case SYS_RES_MEMORY: 538195534Sscottl rman_release_resource(r); 539195534Sscottl return (0); 540195534Sscottl case SYS_RES_IRQ: 541195534Sscottl if (rid != ATA_IRQ_RID) 542277061Ssmh return (ENOENT); 543195534Sscottl return (0); 544195534Sscottl } 545195534Sscottl return (EINVAL); 546195534Sscottl} 547195534Sscottl 548279918Smavint 549195534Sscottlahci_setup_intr(device_t dev, device_t child, struct resource *irq, 550279918Smav int flags, driver_filter_t *filter, driver_intr_t *function, 551279918Smav void *argument, void **cookiep) 552195534Sscottl{ 553195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 554195534Sscottl int unit = (intptr_t)device_get_ivars(child); 555195534Sscottl 556195534Sscottl if (filter != NULL) { 557195534Sscottl printf("ahci.c: we cannot use a filter here\n"); 558195534Sscottl return (EINVAL); 559195534Sscottl } 560195534Sscottl ctlr->interrupt[unit].function = function; 561195534Sscottl ctlr->interrupt[unit].argument = argument; 562195534Sscottl return (0); 563195534Sscottl} 564195534Sscottl 565279918Smavint 566195534Sscottlahci_teardown_intr(device_t dev, device_t child, struct resource *irq, 567279918Smav void *cookie) 568195534Sscottl{ 569195534Sscottl struct ahci_controller *ctlr = device_get_softc(dev); 570195534Sscottl int unit = (intptr_t)device_get_ivars(child); 571195534Sscottl 572195534Sscottl ctlr->interrupt[unit].function = NULL; 573195534Sscottl ctlr->interrupt[unit].argument = NULL; 574195534Sscottl return (0); 575195534Sscottl} 576195534Sscottl 577279918Smavint 578195534Sscottlahci_print_child(device_t dev, device_t child) 579195534Sscottl{ 580238805Smav int retval, channel; 581195534Sscottl 582195534Sscottl retval = bus_print_child_header(dev, child); 583238805Smav channel = (int)(intptr_t)device_get_ivars(child); 584238805Smav if (channel >= 0) 585238805Smav retval += printf(" at channel %d", channel); 586195534Sscottl retval += bus_print_child_footer(dev, child); 587195534Sscottl return (retval); 588195534Sscottl} 589195534Sscottl 590279918Smavint 591208410Smavahci_child_location_str(device_t dev, device_t child, char *buf, 592208410Smav size_t buflen) 593208410Smav{ 594238805Smav int channel; 595208410Smav 596238805Smav channel = (int)(intptr_t)device_get_ivars(child); 597238805Smav if (channel >= 0) 598238805Smav snprintf(buf, buflen, "channel=%d", channel); 599208410Smav return (0); 600208410Smav} 601208410Smav 602279918Smavbus_dma_tag_t 603249346Smavahci_get_dma_tag(device_t dev, device_t child) 604249346Smav{ 605249346Smav struct ahci_controller *ctlr = device_get_softc(dev); 606249346Smav 607249346Smav return (ctlr->dma_tag); 608249346Smav} 609249346Smav 610195534Sscottlstatic int 611195534Sscottlahci_ch_probe(device_t dev) 612195534Sscottl{ 613195534Sscottl 614195534Sscottl device_set_desc_copy(dev, "AHCI channel"); 615281140Smav return (BUS_PROBE_DEFAULT); 616195534Sscottl} 617195534Sscottl 618195534Sscottlstatic int 619195534Sscottlahci_ch_attach(device_t dev) 620195534Sscottl{ 621195534Sscottl struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev)); 622195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 623195534Sscottl struct cam_devq *devq; 624199821Smav int rid, error, i, sata_rev = 0; 625203123Smav u_int32_t version; 626195534Sscottl 627195534Sscottl ch->dev = dev; 628195534Sscottl ch->unit = (intptr_t)device_get_ivars(dev); 629196656Smav ch->caps = ctlr->caps; 630196656Smav ch->caps2 = ctlr->caps2; 631199322Smav ch->quirks = ctlr->quirks; 632279918Smav ch->vendorid = ctlr->vendorid; 633279918Smav ch->deviceid = ctlr->deviceid; 634279918Smav ch->subvendorid = ctlr->subvendorid; 635279918Smav ch->subdeviceid = ctlr->subdeviceid; 636215725Smav ch->numslots = ((ch->caps & AHCI_CAP_NCS) >> AHCI_CAP_NCS_SHIFT) + 1; 637196656Smav mtx_init(&ch->mtx, "AHCI channel lock", NULL, MTX_DEF); 638279918Smav ch->pm_level = 0; 639195534Sscottl resource_int_value(device_get_name(dev), 640195534Sscottl device_get_unit(dev), "pm_level", &ch->pm_level); 641260387Sscottl STAILQ_INIT(&ch->doneq); 642196656Smav if (ch->pm_level > 3) 643196656Smav callout_init_mtx(&ch->pm_timer, &ch->mtx, 0); 644220576Smav callout_init_mtx(&ch->reset_timer, &ch->mtx, 0); 645279918Smav /* JMicron external ports (0) sometimes limited */ 646279918Smav if ((ctlr->quirks & AHCI_Q_SATA1_UNIT0) && ch->unit == 0) 647199821Smav sata_rev = 1; 648203030Smav if (ch->quirks & AHCI_Q_SATA2) 649203030Smav sata_rev = 2; 650195534Sscottl resource_int_value(device_get_name(dev), 651199821Smav device_get_unit(dev), "sata_rev", &sata_rev); 652199821Smav for (i = 0; i < 16; i++) { 653199821Smav ch->user[i].revision = sata_rev; 654199821Smav ch->user[i].mode = 0; 655199821Smav ch->user[i].bytecount = 8192; 656199821Smav ch->user[i].tags = ch->numslots; 657207499Smav ch->user[i].caps = 0; 658199821Smav ch->curr[i] = ch->user[i]; 659207499Smav if (ch->pm_level) { 660207499Smav ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ | 661207499Smav CTS_SATA_CAPS_H_APST | 662207499Smav CTS_SATA_CAPS_D_PMREQ | CTS_SATA_CAPS_D_APST; 663207499Smav } 664220602Smav ch->user[i].caps |= CTS_SATA_CAPS_H_DMAAA | 665220602Smav CTS_SATA_CAPS_H_AN; 666199821Smav } 667238805Smav rid = 0; 668195534Sscottl if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 669195534Sscottl &rid, RF_ACTIVE))) 670195534Sscottl return (ENXIO); 671305798Smav ch->chcaps = ATA_INL(ch->r_mem, AHCI_P_CMD); 672305798Smav version = ATA_INL(ctlr->r_mem, AHCI_VS); 673305798Smav if (version < 0x00010200 && (ctlr->caps & AHCI_CAP_FBSS)) 674305798Smav ch->chcaps |= AHCI_P_CMD_FBSCP; 675305798Smav if (ch->caps2 & AHCI_CAP2_SDS) 676305798Smav ch->chscaps = ATA_INL(ch->r_mem, AHCI_P_DEVSLP); 677305798Smav if (bootverbose) { 678305798Smav device_printf(dev, "Caps:%s%s%s%s%s%s\n", 679305798Smav (ch->chcaps & AHCI_P_CMD_HPCP) ? " HPCP":"", 680305798Smav (ch->chcaps & AHCI_P_CMD_MPSP) ? " MPSP":"", 681305798Smav (ch->chcaps & AHCI_P_CMD_CPD) ? " CPD":"", 682305798Smav (ch->chcaps & AHCI_P_CMD_ESP) ? " ESP":"", 683305798Smav (ch->chcaps & AHCI_P_CMD_FBSCP) ? " FBSCP":"", 684305798Smav (ch->chscaps & AHCI_P_DEVSLP_DSP) ? " DSP":""); 685305798Smav } 686195534Sscottl ahci_dmainit(dev); 687195534Sscottl ahci_slotsalloc(dev); 688279918Smav mtx_lock(&ch->mtx); 689208375Smav ahci_ch_init(dev); 690195534Sscottl rid = ATA_IRQ_RID; 691195534Sscottl if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, 692195534Sscottl &rid, RF_SHAREABLE | RF_ACTIVE))) { 693195534Sscottl device_printf(dev, "Unable to map interrupt\n"); 694208813Smav error = ENXIO; 695208813Smav goto err0; 696195534Sscottl } 697195534Sscottl if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL, 698260387Sscottl ctlr->direct ? ahci_ch_intr_direct : ahci_ch_intr, 699279918Smav ch, &ch->ih))) { 700195534Sscottl device_printf(dev, "Unable to setup interrupt\n"); 701195534Sscottl error = ENXIO; 702195534Sscottl goto err1; 703195534Sscottl } 704195534Sscottl /* Create the device queue for our SIM. */ 705195534Sscottl devq = cam_simq_alloc(ch->numslots); 706195534Sscottl if (devq == NULL) { 707195534Sscottl device_printf(dev, "Unable to allocate simq\n"); 708195534Sscottl error = ENOMEM; 709195534Sscottl goto err1; 710195534Sscottl } 711195534Sscottl /* Construct SIM entry */ 712195534Sscottl ch->sim = cam_sim_alloc(ahciaction, ahcipoll, "ahcich", ch, 713279918Smav device_get_unit(dev), (struct mtx *)&ch->mtx, 714313446Smav (ch->quirks & AHCI_Q_NOCCS) ? 1 : min(2, ch->numslots), 715199278Smav (ch->caps & AHCI_CAP_SNCQ) ? ch->numslots : 0, 716199278Smav devq); 717195534Sscottl if (ch->sim == NULL) { 718208813Smav cam_simq_free(devq); 719195534Sscottl device_printf(dev, "unable to allocate sim\n"); 720195534Sscottl error = ENOMEM; 721208813Smav goto err1; 722195534Sscottl } 723195534Sscottl if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) { 724195534Sscottl device_printf(dev, "unable to register xpt bus\n"); 725195534Sscottl error = ENXIO; 726195534Sscottl goto err2; 727195534Sscottl } 728195534Sscottl if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim), 729195534Sscottl CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 730195534Sscottl device_printf(dev, "unable to create path\n"); 731195534Sscottl error = ENXIO; 732195534Sscottl goto err3; 733195534Sscottl } 734196656Smav if (ch->pm_level > 3) { 735196656Smav callout_reset(&ch->pm_timer, 736196656Smav (ch->pm_level == 4) ? hz / 1000 : hz / 8, 737279918Smav ahci_ch_pm, ch); 738196656Smav } 739195534Sscottl mtx_unlock(&ch->mtx); 740195534Sscottl return (0); 741195534Sscottl 742195534Sscottlerr3: 743195534Sscottl xpt_bus_deregister(cam_sim_path(ch->sim)); 744195534Sscottlerr2: 745195534Sscottl cam_sim_free(ch->sim, /*free_devq*/TRUE); 746195534Sscottlerr1: 747195534Sscottl bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 748208813Smaverr0: 749195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem); 750195534Sscottl mtx_unlock(&ch->mtx); 751214325Smav mtx_destroy(&ch->mtx); 752195534Sscottl return (error); 753195534Sscottl} 754195534Sscottl 755195534Sscottlstatic int 756195534Sscottlahci_ch_detach(device_t dev) 757195534Sscottl{ 758195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 759195534Sscottl 760195534Sscottl mtx_lock(&ch->mtx); 761195534Sscottl xpt_async(AC_LOST_DEVICE, ch->path, NULL); 762220576Smav /* Forget about reset. */ 763220576Smav if (ch->resetting) { 764220576Smav ch->resetting = 0; 765220576Smav xpt_release_simq(ch->sim, TRUE); 766220576Smav } 767195534Sscottl xpt_free_path(ch->path); 768195534Sscottl xpt_bus_deregister(cam_sim_path(ch->sim)); 769195534Sscottl cam_sim_free(ch->sim, /*free_devq*/TRUE); 770195534Sscottl mtx_unlock(&ch->mtx); 771195534Sscottl 772196656Smav if (ch->pm_level > 3) 773196656Smav callout_drain(&ch->pm_timer); 774220576Smav callout_drain(&ch->reset_timer); 775195534Sscottl bus_teardown_intr(dev, ch->r_irq, ch->ih); 776195534Sscottl bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 777195534Sscottl 778208375Smav ahci_ch_deinit(dev); 779195534Sscottl ahci_slotsfree(dev); 780195534Sscottl ahci_dmafini(dev); 781195534Sscottl 782195534Sscottl bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem); 783195534Sscottl mtx_destroy(&ch->mtx); 784195534Sscottl return (0); 785195534Sscottl} 786195534Sscottl 787195534Sscottlstatic int 788208375Smavahci_ch_init(device_t dev) 789195534Sscottl{ 790195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 791208375Smav uint64_t work; 792195534Sscottl 793208375Smav /* Disable port interrupts */ 794208375Smav ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 795208375Smav /* Setup work areas */ 796208375Smav work = ch->dma.work_bus + AHCI_CL_OFFSET; 797208375Smav ATA_OUTL(ch->r_mem, AHCI_P_CLB, work & 0xffffffff); 798208375Smav ATA_OUTL(ch->r_mem, AHCI_P_CLBU, work >> 32); 799208375Smav work = ch->dma.rfis_bus; 800208375Smav ATA_OUTL(ch->r_mem, AHCI_P_FB, work & 0xffffffff); 801208375Smav ATA_OUTL(ch->r_mem, AHCI_P_FBU, work >> 32); 802208375Smav /* Activate the channel and power/spin up device */ 803208375Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, 804208375Smav (AHCI_P_CMD_ACTIVE | AHCI_P_CMD_POD | AHCI_P_CMD_SUD | 805208375Smav ((ch->pm_level == 2 || ch->pm_level == 3) ? AHCI_P_CMD_ALPE : 0) | 806208375Smav ((ch->pm_level > 2) ? AHCI_P_CMD_ASP : 0 ))); 807279918Smav ahci_start_fr(ch); 808279918Smav ahci_start(ch, 1); 809208375Smav return (0); 810208375Smav} 811208375Smav 812208375Smavstatic int 813208375Smavahci_ch_deinit(device_t dev) 814208375Smav{ 815208375Smav struct ahci_channel *ch = device_get_softc(dev); 816208375Smav 817195534Sscottl /* Disable port interrupts. */ 818195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 819195534Sscottl /* Reset command register. */ 820279918Smav ahci_stop(ch); 821279918Smav ahci_stop_fr(ch); 822195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, 0); 823195534Sscottl /* Allow everything, including partial and slumber modes. */ 824195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 0); 825195534Sscottl /* Request slumber mode transition and give some time to get there. */ 826195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, AHCI_P_CMD_SLUMBER); 827195534Sscottl DELAY(100); 828195534Sscottl /* Disable PHY. */ 829195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE); 830195534Sscottl return (0); 831195534Sscottl} 832195534Sscottl 833195534Sscottlstatic int 834208375Smavahci_ch_suspend(device_t dev) 835208375Smav{ 836208375Smav struct ahci_channel *ch = device_get_softc(dev); 837208375Smav 838208375Smav mtx_lock(&ch->mtx); 839208375Smav xpt_freeze_simq(ch->sim, 1); 840220576Smav /* Forget about reset. */ 841220576Smav if (ch->resetting) { 842220576Smav ch->resetting = 0; 843220576Smav callout_stop(&ch->reset_timer); 844220576Smav xpt_release_simq(ch->sim, TRUE); 845220576Smav } 846208375Smav while (ch->oslots) 847208375Smav msleep(ch, &ch->mtx, PRIBIO, "ahcisusp", hz/100); 848208375Smav ahci_ch_deinit(dev); 849208375Smav mtx_unlock(&ch->mtx); 850208375Smav return (0); 851208375Smav} 852208375Smav 853208375Smavstatic int 854195534Sscottlahci_ch_resume(device_t dev) 855195534Sscottl{ 856195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 857195534Sscottl 858208375Smav mtx_lock(&ch->mtx); 859208375Smav ahci_ch_init(dev); 860279918Smav ahci_reset(ch); 861208375Smav xpt_release_simq(ch->sim, TRUE); 862208375Smav mtx_unlock(&ch->mtx); 863195534Sscottl return (0); 864195534Sscottl} 865195534Sscottl 866195534Sscottldevclass_t ahcich_devclass; 867195534Sscottlstatic device_method_t ahcich_methods[] = { 868195534Sscottl DEVMETHOD(device_probe, ahci_ch_probe), 869195534Sscottl DEVMETHOD(device_attach, ahci_ch_attach), 870195534Sscottl DEVMETHOD(device_detach, ahci_ch_detach), 871195534Sscottl DEVMETHOD(device_suspend, ahci_ch_suspend), 872195534Sscottl DEVMETHOD(device_resume, ahci_ch_resume), 873278405Smarius DEVMETHOD_END 874195534Sscottl}; 875195534Sscottlstatic driver_t ahcich_driver = { 876195534Sscottl "ahcich", 877195534Sscottl ahcich_methods, 878195534Sscottl sizeof(struct ahci_channel) 879195534Sscottl}; 880278405SmariusDRIVER_MODULE(ahcich, ahci, ahcich_driver, ahcich_devclass, NULL, NULL); 881195534Sscottl 882195534Sscottlstruct ahci_dc_cb_args { 883195534Sscottl bus_addr_t maddr; 884195534Sscottl int error; 885195534Sscottl}; 886195534Sscottl 887195534Sscottlstatic void 888195534Sscottlahci_dmainit(device_t dev) 889195534Sscottl{ 890195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 891195534Sscottl struct ahci_dc_cb_args dcba; 892203123Smav size_t rfsize; 893195534Sscottl 894195534Sscottl /* Command area. */ 895195534Sscottl if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0, 896249346Smav BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 897195534Sscottl NULL, NULL, AHCI_WORK_SIZE, 1, AHCI_WORK_SIZE, 898195534Sscottl 0, NULL, NULL, &ch->dma.work_tag)) 899195534Sscottl goto error; 900248687Smav if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 901248687Smav BUS_DMA_ZERO, &ch->dma.work_map)) 902195534Sscottl goto error; 903195534Sscottl if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work, 904195534Sscottl AHCI_WORK_SIZE, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) { 905195534Sscottl bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 906195534Sscottl goto error; 907195534Sscottl } 908195534Sscottl ch->dma.work_bus = dcba.maddr; 909195534Sscottl /* FIS receive area. */ 910203123Smav if (ch->chcaps & AHCI_P_CMD_FBSCP) 911203123Smav rfsize = 4096; 912203123Smav else 913203123Smav rfsize = 256; 914203123Smav if (bus_dma_tag_create(bus_get_dma_tag(dev), rfsize, 0, 915249346Smav BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 916203123Smav NULL, NULL, rfsize, 1, rfsize, 917195534Sscottl 0, NULL, NULL, &ch->dma.rfis_tag)) 918195534Sscottl goto error; 919195534Sscottl if (bus_dmamem_alloc(ch->dma.rfis_tag, (void **)&ch->dma.rfis, 0, 920195534Sscottl &ch->dma.rfis_map)) 921195534Sscottl goto error; 922195534Sscottl if (bus_dmamap_load(ch->dma.rfis_tag, ch->dma.rfis_map, ch->dma.rfis, 923203123Smav rfsize, ahci_dmasetupc_cb, &dcba, 0) || dcba.error) { 924195534Sscottl bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map); 925195534Sscottl goto error; 926195534Sscottl } 927195534Sscottl ch->dma.rfis_bus = dcba.maddr; 928195534Sscottl /* Data area. */ 929195534Sscottl if (bus_dma_tag_create(bus_get_dma_tag(dev), 2, 0, 930249346Smav BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 931195534Sscottl NULL, NULL, 932195534Sscottl AHCI_SG_ENTRIES * PAGE_SIZE * ch->numslots, 933195534Sscottl AHCI_SG_ENTRIES, AHCI_PRD_MAX, 934195534Sscottl 0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) { 935195534Sscottl goto error; 936195534Sscottl } 937195534Sscottl return; 938195534Sscottl 939195534Sscottlerror: 940195534Sscottl device_printf(dev, "WARNING - DMA initialization failed\n"); 941195534Sscottl ahci_dmafini(dev); 942195534Sscottl} 943195534Sscottl 944195534Sscottlstatic void 945195534Sscottlahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 946195534Sscottl{ 947195534Sscottl struct ahci_dc_cb_args *dcba = (struct ahci_dc_cb_args *)xsc; 948195534Sscottl 949195534Sscottl if (!(dcba->error = error)) 950195534Sscottl dcba->maddr = segs[0].ds_addr; 951195534Sscottl} 952195534Sscottl 953195534Sscottlstatic void 954195534Sscottlahci_dmafini(device_t dev) 955195534Sscottl{ 956195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 957195534Sscottl 958195534Sscottl if (ch->dma.data_tag) { 959195534Sscottl bus_dma_tag_destroy(ch->dma.data_tag); 960195534Sscottl ch->dma.data_tag = NULL; 961195534Sscottl } 962195534Sscottl if (ch->dma.rfis_bus) { 963195534Sscottl bus_dmamap_unload(ch->dma.rfis_tag, ch->dma.rfis_map); 964195534Sscottl bus_dmamem_free(ch->dma.rfis_tag, ch->dma.rfis, ch->dma.rfis_map); 965195534Sscottl ch->dma.rfis_bus = 0; 966195534Sscottl ch->dma.rfis = NULL; 967195534Sscottl } 968195534Sscottl if (ch->dma.work_bus) { 969195534Sscottl bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map); 970195534Sscottl bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 971195534Sscottl ch->dma.work_bus = 0; 972195534Sscottl ch->dma.work = NULL; 973195534Sscottl } 974195534Sscottl if (ch->dma.work_tag) { 975195534Sscottl bus_dma_tag_destroy(ch->dma.work_tag); 976195534Sscottl ch->dma.work_tag = NULL; 977195534Sscottl } 978195534Sscottl} 979195534Sscottl 980195534Sscottlstatic void 981195534Sscottlahci_slotsalloc(device_t dev) 982195534Sscottl{ 983195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 984195534Sscottl int i; 985195534Sscottl 986195534Sscottl /* Alloc and setup command/dma slots */ 987195534Sscottl bzero(ch->slot, sizeof(ch->slot)); 988195534Sscottl for (i = 0; i < ch->numslots; i++) { 989195534Sscottl struct ahci_slot *slot = &ch->slot[i]; 990195534Sscottl 991279918Smav slot->ch = ch; 992195534Sscottl slot->slot = i; 993195534Sscottl slot->state = AHCI_SLOT_EMPTY; 994195534Sscottl slot->ccb = NULL; 995195534Sscottl callout_init_mtx(&slot->timeout, &ch->mtx, 0); 996195534Sscottl 997195534Sscottl if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map)) 998195534Sscottl device_printf(ch->dev, "FAILURE - create data_map\n"); 999195534Sscottl } 1000195534Sscottl} 1001195534Sscottl 1002195534Sscottlstatic void 1003195534Sscottlahci_slotsfree(device_t dev) 1004195534Sscottl{ 1005195534Sscottl struct ahci_channel *ch = device_get_softc(dev); 1006195534Sscottl int i; 1007195534Sscottl 1008195534Sscottl /* Free all dma slots */ 1009195534Sscottl for (i = 0; i < ch->numslots; i++) { 1010195534Sscottl struct ahci_slot *slot = &ch->slot[i]; 1011195534Sscottl 1012196656Smav callout_drain(&slot->timeout); 1013195534Sscottl if (slot->dma.data_map) { 1014195534Sscottl bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map); 1015195534Sscottl slot->dma.data_map = NULL; 1016195534Sscottl } 1017195534Sscottl } 1018195534Sscottl} 1019195534Sscottl 1020220657Smavstatic int 1021279918Smavahci_phy_check_events(struct ahci_channel *ch, u_int32_t serr) 1022195534Sscottl{ 1023195534Sscottl 1024220657Smav if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) || 1025220657Smav ((ch->pm_level != 0 || ch->listening) && (serr & ATA_SE_EXCHANGED))) { 1026195534Sscottl u_int32_t status = ATA_INL(ch->r_mem, AHCI_P_SSTS); 1027203108Smav union ccb *ccb; 1028203108Smav 1029203165Smav if (bootverbose) { 1030220657Smav if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE) 1031279918Smav device_printf(ch->dev, "CONNECT requested\n"); 1032220657Smav else 1033279918Smav device_printf(ch->dev, "DISCONNECT requested\n"); 1034195534Sscottl } 1035279918Smav ahci_reset(ch); 1036203108Smav if ((ccb = xpt_alloc_ccb_nowait()) == NULL) 1037220657Smav return (0); 1038203108Smav if (xpt_create_path(&ccb->ccb_h.path, NULL, 1039203108Smav cam_sim_path(ch->sim), 1040203108Smav CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 1041203108Smav xpt_free_ccb(ccb); 1042220657Smav return (0); 1043203108Smav } 1044203108Smav xpt_rescan(ccb); 1045220657Smav return (1); 1046195534Sscottl } 1047220657Smav return (0); 1048195534Sscottl} 1049195534Sscottl 1050195534Sscottlstatic void 1051279918Smavahci_cpd_check_events(struct ahci_channel *ch) 1052220657Smav{ 1053220657Smav u_int32_t status; 1054220657Smav union ccb *ccb; 1055279918Smav device_t dev; 1056220657Smav 1057220657Smav if (ch->pm_level == 0) 1058220657Smav return; 1059220657Smav 1060220657Smav status = ATA_INL(ch->r_mem, AHCI_P_CMD); 1061220657Smav if ((status & AHCI_P_CMD_CPD) == 0) 1062220657Smav return; 1063220657Smav 1064220657Smav if (bootverbose) { 1065279918Smav dev = ch->dev; 1066220657Smav if (status & AHCI_P_CMD_CPS) { 1067220657Smav device_printf(dev, "COLD CONNECT requested\n"); 1068220657Smav } else 1069220657Smav device_printf(dev, "COLD DISCONNECT requested\n"); 1070220657Smav } 1071279918Smav ahci_reset(ch); 1072220657Smav if ((ccb = xpt_alloc_ccb_nowait()) == NULL) 1073220657Smav return; 1074220657Smav if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(ch->sim), 1075220657Smav CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 1076220657Smav xpt_free_ccb(ccb); 1077220657Smav return; 1078220657Smav } 1079220657Smav xpt_rescan(ccb); 1080220657Smav} 1081220657Smav 1082220657Smavstatic void 1083279918Smavahci_notify_events(struct ahci_channel *ch, u_int32_t status) 1084196656Smav{ 1085196656Smav struct cam_path *dpath; 1086196656Smav int i; 1087196656Smav 1088200196Smav if (ch->caps & AHCI_CAP_SSNTF) 1089200196Smav ATA_OUTL(ch->r_mem, AHCI_P_SNTF, status); 1090196656Smav if (bootverbose) 1091279918Smav device_printf(ch->dev, "SNTF 0x%04x\n", status); 1092196656Smav for (i = 0; i < 16; i++) { 1093196656Smav if ((status & (1 << i)) == 0) 1094196656Smav continue; 1095196656Smav if (xpt_create_path(&dpath, NULL, 1096196656Smav xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) { 1097196656Smav xpt_async(AC_SCSI_AEN, dpath, NULL); 1098196656Smav xpt_free_path(dpath); 1099196656Smav } 1100196656Smav } 1101196656Smav} 1102196656Smav 1103196656Smavstatic void 1104260387Sscottlahci_done(struct ahci_channel *ch, union ccb *ccb) 1105195534Sscottl{ 1106260387Sscottl 1107260387Sscottl mtx_assert(&ch->mtx, MA_OWNED); 1108260387Sscottl if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0 || 1109260387Sscottl ch->batch == 0) { 1110260387Sscottl xpt_done(ccb); 1111260387Sscottl return; 1112260387Sscottl } 1113260387Sscottl 1114260387Sscottl STAILQ_INSERT_TAIL(&ch->doneq, &ccb->ccb_h, sim_links.stqe); 1115260387Sscottl} 1116260387Sscottl 1117260387Sscottlstatic void 1118260387Sscottlahci_ch_intr(void *arg) 1119260387Sscottl{ 1120279918Smav struct ahci_channel *ch = (struct ahci_channel *)arg; 1121260387Sscottl uint32_t istatus; 1122195534Sscottl 1123260387Sscottl /* Read interrupt statuses. */ 1124260387Sscottl istatus = ATA_INL(ch->r_mem, AHCI_P_IS); 1125260387Sscottl 1126195534Sscottl mtx_lock(&ch->mtx); 1127260387Sscottl ahci_ch_intr_main(ch, istatus); 1128195534Sscottl mtx_unlock(&ch->mtx); 1129195534Sscottl} 1130195534Sscottl 1131195534Sscottlstatic void 1132260387Sscottlahci_ch_intr_direct(void *arg) 1133260387Sscottl{ 1134279918Smav struct ahci_channel *ch = (struct ahci_channel *)arg; 1135260387Sscottl struct ccb_hdr *ccb_h; 1136260387Sscottl uint32_t istatus; 1137276020Ssmh STAILQ_HEAD(, ccb_hdr) tmp_doneq = STAILQ_HEAD_INITIALIZER(tmp_doneq); 1138260387Sscottl 1139260387Sscottl /* Read interrupt statuses. */ 1140260387Sscottl istatus = ATA_INL(ch->r_mem, AHCI_P_IS); 1141260387Sscottl 1142260387Sscottl mtx_lock(&ch->mtx); 1143260387Sscottl ch->batch = 1; 1144260387Sscottl ahci_ch_intr_main(ch, istatus); 1145260387Sscottl ch->batch = 0; 1146276020Ssmh /* 1147276020Ssmh * Prevent the possibility of issues caused by processing the queue 1148276020Ssmh * while unlocked below by moving the contents to a local queue. 1149276020Ssmh */ 1150276020Ssmh STAILQ_CONCAT(&tmp_doneq, &ch->doneq); 1151260387Sscottl mtx_unlock(&ch->mtx); 1152276020Ssmh while ((ccb_h = STAILQ_FIRST(&tmp_doneq)) != NULL) { 1153276020Ssmh STAILQ_REMOVE_HEAD(&tmp_doneq, sim_links.stqe); 1154260387Sscottl xpt_done_direct((union ccb *)ccb_h); 1155260387Sscottl } 1156260387Sscottl} 1157260387Sscottl 1158260387Sscottlstatic void 1159196656Smavahci_ch_pm(void *arg) 1160196656Smav{ 1161279918Smav struct ahci_channel *ch = (struct ahci_channel *)arg; 1162196656Smav uint32_t work; 1163196656Smav 1164196656Smav if (ch->numrslots != 0) 1165196656Smav return; 1166196656Smav work = ATA_INL(ch->r_mem, AHCI_P_CMD); 1167196656Smav if (ch->pm_level == 4) 1168196656Smav work |= AHCI_P_CMD_PARTIAL; 1169196656Smav else 1170196656Smav work |= AHCI_P_CMD_SLUMBER; 1171196656Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, work); 1172196656Smav} 1173196656Smav 1174196656Smavstatic void 1175260387Sscottlahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus) 1176195534Sscottl{ 1177260387Sscottl uint32_t cstatus, serr = 0, sntf = 0, ok, err; 1178195534Sscottl enum ahci_err_type et; 1179220657Smav int i, ccs, port, reset = 0; 1180195534Sscottl 1181260387Sscottl /* Clear interrupt statuses. */ 1182195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IS, istatus); 1183195534Sscottl /* Read command statuses. */ 1184248698Smav if (ch->numtslots != 0) 1185248698Smav cstatus = ATA_INL(ch->r_mem, AHCI_P_SACT); 1186248698Smav else 1187248698Smav cstatus = 0; 1188248698Smav if (ch->numrslots != ch->numtslots) 1189248698Smav cstatus |= ATA_INL(ch->r_mem, AHCI_P_CI); 1190248698Smav /* Read SNTF in one of possible ways. */ 1191248704Smav if ((istatus & AHCI_P_IX_SDB) && 1192248704Smav (ch->pm_present || ch->curr[0].atapi != 0)) { 1193200196Smav if (ch->caps & AHCI_CAP_SSNTF) 1194200196Smav sntf = ATA_INL(ch->r_mem, AHCI_P_SNTF); 1195203123Smav else if (ch->fbs_enabled) { 1196200196Smav u_int8_t *fis = ch->dma.rfis + 0x58; 1197200196Smav 1198203123Smav for (i = 0; i < 16; i++) { 1199203123Smav if (fis[1] & 0x80) { 1200203123Smav fis[1] &= 0x7f; 1201203123Smav sntf |= 1 << i; 1202203123Smav } 1203203123Smav fis += 256; 1204203123Smav } 1205203123Smav } else { 1206203123Smav u_int8_t *fis = ch->dma.rfis + 0x58; 1207203123Smav 1208200196Smav if (fis[1] & 0x80) 1209200196Smav sntf = (1 << (fis[1] & 0x0f)); 1210200196Smav } 1211200196Smav } 1212195534Sscottl /* Process PHY events */ 1213198319Smav if (istatus & (AHCI_P_IX_PC | AHCI_P_IX_PRC | AHCI_P_IX_OF | 1214198319Smav AHCI_P_IX_IF | AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) { 1215198319Smav serr = ATA_INL(ch->r_mem, AHCI_P_SERR); 1216198319Smav if (serr) { 1217198319Smav ATA_OUTL(ch->r_mem, AHCI_P_SERR, serr); 1218279918Smav reset = ahci_phy_check_events(ch, serr); 1219198319Smav } 1220198319Smav } 1221220657Smav /* Process cold presence detection events */ 1222220657Smav if ((istatus & AHCI_P_IX_CPD) && !reset) 1223279918Smav ahci_cpd_check_events(ch); 1224195534Sscottl /* Process command errors */ 1225198319Smav if (istatus & (AHCI_P_IX_OF | AHCI_P_IX_IF | 1226198319Smav AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) { 1227313446Smav if (ch->quirks & AHCI_Q_NOCCS) { 1228313446Smav /* 1229313446Smav * ASMedia chips sometimes report failed commands as 1230313446Smav * completed. Count all running commands as failed. 1231313446Smav */ 1232313446Smav cstatus |= ch->rslots; 1233313446Smav 1234313446Smav /* They also report wrong CCS, so try to guess one. */ 1235313446Smav ccs = powerof2(cstatus) ? ffs(cstatus) - 1 : -1; 1236313446Smav } else { 1237313446Smav ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & 1238313446Smav AHCI_P_CMD_CCS_MASK) >> AHCI_P_CMD_CCS_SHIFT; 1239313446Smav } 1240203123Smav//device_printf(dev, "%s ERROR is %08x cs %08x ss %08x rs %08x tfd %02x serr %08x fbs %08x ccs %d\n", 1241203123Smav// __func__, istatus, cstatus, sstatus, ch->rslots, ATA_INL(ch->r_mem, AHCI_P_TFD), 1242203123Smav// serr, ATA_INL(ch->r_mem, AHCI_P_FBS), ccs); 1243203123Smav port = -1; 1244203123Smav if (ch->fbs_enabled) { 1245203123Smav uint32_t fbs = ATA_INL(ch->r_mem, AHCI_P_FBS); 1246203123Smav if (fbs & AHCI_P_FBS_SDE) { 1247203123Smav port = (fbs & AHCI_P_FBS_DWE) 1248203123Smav >> AHCI_P_FBS_DWE_SHIFT; 1249203123Smav } else { 1250203123Smav for (i = 0; i < 16; i++) { 1251203123Smav if (ch->numrslotspd[i] == 0) 1252203123Smav continue; 1253203123Smav if (port == -1) 1254203123Smav port = i; 1255203123Smav else if (port != i) { 1256203123Smav port = -2; 1257203123Smav break; 1258203123Smav } 1259203123Smav } 1260203123Smav } 1261203123Smav } 1262248698Smav err = ch->rslots & cstatus; 1263195534Sscottl } else { 1264195534Sscottl ccs = 0; 1265195534Sscottl err = 0; 1266203123Smav port = -1; 1267195534Sscottl } 1268195534Sscottl /* Complete all successfull commands. */ 1269248698Smav ok = ch->rslots & ~cstatus; 1270195534Sscottl for (i = 0; i < ch->numslots; i++) { 1271195534Sscottl if ((ok >> i) & 1) 1272195534Sscottl ahci_end_transaction(&ch->slot[i], AHCI_ERR_NONE); 1273195534Sscottl } 1274195534Sscottl /* On error, complete the rest of commands with error statuses. */ 1275195534Sscottl if (err) { 1276195534Sscottl if (ch->frozen) { 1277195534Sscottl union ccb *fccb = ch->frozen; 1278195534Sscottl ch->frozen = NULL; 1279195534Sscottl fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 1280198319Smav if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 1281198319Smav xpt_freeze_devq(fccb->ccb_h.path, 1); 1282198319Smav fccb->ccb_h.status |= CAM_DEV_QFRZN; 1283198319Smav } 1284260387Sscottl ahci_done(ch, fccb); 1285195534Sscottl } 1286195534Sscottl for (i = 0; i < ch->numslots; i++) { 1287195534Sscottl /* XXX: reqests in loading state. */ 1288195534Sscottl if (((err >> i) & 1) == 0) 1289195534Sscottl continue; 1290203123Smav if (port >= 0 && 1291203123Smav ch->slot[i].ccb->ccb_h.target_id != port) 1292203123Smav continue; 1293198390Smav if (istatus & AHCI_P_IX_TFE) { 1294203123Smav if (port != -2) { 1295195534Sscottl /* Task File Error */ 1296203123Smav if (ch->numtslotspd[ 1297203123Smav ch->slot[i].ccb->ccb_h.target_id] == 0) { 1298195534Sscottl /* Untagged operation. */ 1299195534Sscottl if (i == ccs) 1300195534Sscottl et = AHCI_ERR_TFE; 1301195534Sscottl else 1302195534Sscottl et = AHCI_ERR_INNOCENT; 1303195534Sscottl } else { 1304195534Sscottl /* Tagged operation. */ 1305195534Sscottl et = AHCI_ERR_NCQ; 1306195534Sscottl } 1307203123Smav } else { 1308203123Smav et = AHCI_ERR_TFE; 1309203123Smav ch->fatalerr = 1; 1310203123Smav } 1311198390Smav } else if (istatus & AHCI_P_IX_IF) { 1312203123Smav if (ch->numtslots == 0 && i != ccs && port != -2) 1313198390Smav et = AHCI_ERR_INNOCENT; 1314198390Smav else 1315198390Smav et = AHCI_ERR_SATA; 1316195534Sscottl } else 1317195534Sscottl et = AHCI_ERR_INVALID; 1318195534Sscottl ahci_end_transaction(&ch->slot[i], et); 1319195534Sscottl } 1320203123Smav /* 1321203123Smav * We can't reinit port if there are some other 1322203123Smav * commands active, use resume to complete them. 1323203123Smav */ 1324220565Smav if (ch->rslots != 0 && !ch->recoverycmd) 1325203123Smav ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | AHCI_P_FBS_DEC); 1326195534Sscottl } 1327196656Smav /* Process NOTIFY events */ 1328196907Smav if (sntf) 1329279918Smav ahci_notify_events(ch, sntf); 1330195534Sscottl} 1331195534Sscottl 1332195534Sscottl/* Must be called with channel locked. */ 1333195534Sscottlstatic int 1334279918Smavahci_check_collision(struct ahci_channel *ch, union ccb *ccb) 1335195534Sscottl{ 1336203123Smav int t = ccb->ccb_h.target_id; 1337195534Sscottl 1338195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1339195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1340199747Smav /* Tagged command while we have no supported tag free. */ 1341199747Smav if (((~ch->oslots) & (0xffffffff >> (32 - 1342203123Smav ch->curr[t].tags))) == 0) 1343199747Smav return (1); 1344203123Smav /* If we have FBS */ 1345203123Smav if (ch->fbs_enabled) { 1346203123Smav /* Tagged command while untagged are active. */ 1347203123Smav if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0) 1348203123Smav return (1); 1349203123Smav } else { 1350203123Smav /* Tagged command while untagged are active. */ 1351203123Smav if (ch->numrslots != 0 && ch->numtslots == 0) 1352203123Smav return (1); 1353203123Smav /* Tagged command while tagged to other target is active. */ 1354203123Smav if (ch->numtslots != 0 && 1355203123Smav ch->taggedtarget != ccb->ccb_h.target_id) 1356203123Smav return (1); 1357203123Smav } 1358195534Sscottl } else { 1359203123Smav /* If we have FBS */ 1360203123Smav if (ch->fbs_enabled) { 1361203123Smav /* Untagged command while tagged are active. */ 1362203123Smav if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0) 1363203123Smav return (1); 1364203123Smav } else { 1365203123Smav /* Untagged command while tagged are active. */ 1366203123Smav if (ch->numrslots != 0 && ch->numtslots != 0) 1367203123Smav return (1); 1368203123Smav } 1369195534Sscottl } 1370195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1371195534Sscottl (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) { 1372195534Sscottl /* Atomic command while anything active. */ 1373195534Sscottl if (ch->numrslots != 0) 1374195534Sscottl return (1); 1375195534Sscottl } 1376195534Sscottl /* We have some atomic command running. */ 1377195534Sscottl if (ch->aslots != 0) 1378195534Sscottl return (1); 1379195534Sscottl return (0); 1380195534Sscottl} 1381195534Sscottl 1382195534Sscottl/* Must be called with channel locked. */ 1383195534Sscottlstatic void 1384279918Smavahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb) 1385195534Sscottl{ 1386195534Sscottl struct ahci_slot *slot; 1387199747Smav int tag, tags; 1388195534Sscottl 1389195534Sscottl /* Choose empty slot. */ 1390199747Smav tags = ch->numslots; 1391199747Smav if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1392199747Smav (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) 1393199747Smav tags = ch->curr[ccb->ccb_h.target_id].tags; 1394279918Smav if (ch->lastslot + 1 < tags) 1395279918Smav tag = ffs(~(ch->oslots >> (ch->lastslot + 1))); 1396279918Smav else 1397279918Smav tag = 0; 1398279918Smav if (tag == 0 || tag + ch->lastslot >= tags) 1399279918Smav tag = ffs(~ch->oslots) - 1; 1400279918Smav else 1401279918Smav tag += ch->lastslot; 1402195534Sscottl ch->lastslot = tag; 1403195534Sscottl /* Occupy chosen slot. */ 1404195534Sscottl slot = &ch->slot[tag]; 1405195534Sscottl slot->ccb = ccb; 1406196656Smav /* Stop PM timer. */ 1407196656Smav if (ch->numrslots == 0 && ch->pm_level > 3) 1408196656Smav callout_stop(&ch->pm_timer); 1409195534Sscottl /* Update channel stats. */ 1410279918Smav ch->oslots |= (1 << tag); 1411195534Sscottl ch->numrslots++; 1412203123Smav ch->numrslotspd[ccb->ccb_h.target_id]++; 1413195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1414195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1415195534Sscottl ch->numtslots++; 1416203123Smav ch->numtslotspd[ccb->ccb_h.target_id]++; 1417195534Sscottl ch->taggedtarget = ccb->ccb_h.target_id; 1418195534Sscottl } 1419195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1420195534Sscottl (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) 1421279918Smav ch->aslots |= (1 << tag); 1422195534Sscottl if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 1423195534Sscottl slot->state = AHCI_SLOT_LOADING; 1424246713Skib bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb, 1425246713Skib ahci_dmasetprd, slot, 0); 1426279918Smav } else { 1427279918Smav slot->dma.nsegs = 0; 1428195534Sscottl ahci_execute_transaction(slot); 1429279918Smav } 1430195534Sscottl} 1431195534Sscottl 1432195534Sscottl/* Locked by busdma engine. */ 1433195534Sscottlstatic void 1434195534Sscottlahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 1435195534Sscottl{ 1436195534Sscottl struct ahci_slot *slot = arg; 1437279918Smav struct ahci_channel *ch = slot->ch; 1438195534Sscottl struct ahci_cmd_tab *ctp; 1439195534Sscottl struct ahci_dma_prd *prd; 1440195534Sscottl int i; 1441195534Sscottl 1442195534Sscottl if (error) { 1443279918Smav device_printf(ch->dev, "DMA load error\n"); 1444195534Sscottl ahci_end_transaction(slot, AHCI_ERR_INVALID); 1445195534Sscottl return; 1446195534Sscottl } 1447195534Sscottl KASSERT(nsegs <= AHCI_SG_ENTRIES, ("too many DMA segment entries\n")); 1448195534Sscottl /* Get a piece of the workspace for this request */ 1449195534Sscottl ctp = (struct ahci_cmd_tab *) 1450195534Sscottl (ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot)); 1451195534Sscottl /* Fill S/G table */ 1452195534Sscottl prd = &ctp->prd_tab[0]; 1453195534Sscottl for (i = 0; i < nsegs; i++) { 1454195534Sscottl prd[i].dba = htole64(segs[i].ds_addr); 1455195534Sscottl prd[i].dbc = htole32((segs[i].ds_len - 1) & AHCI_PRD_MASK); 1456195534Sscottl } 1457195534Sscottl slot->dma.nsegs = nsegs; 1458195534Sscottl bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 1459195534Sscottl ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ? 1460195534Sscottl BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE)); 1461195534Sscottl ahci_execute_transaction(slot); 1462195534Sscottl} 1463195534Sscottl 1464195534Sscottl/* Must be called with channel locked. */ 1465195534Sscottlstatic void 1466195534Sscottlahci_execute_transaction(struct ahci_slot *slot) 1467195534Sscottl{ 1468279918Smav struct ahci_channel *ch = slot->ch; 1469195534Sscottl struct ahci_cmd_tab *ctp; 1470195534Sscottl struct ahci_cmd_list *clp; 1471195534Sscottl union ccb *ccb = slot->ccb; 1472195534Sscottl int port = ccb->ccb_h.target_id & 0x0f; 1473222304Smav int fis_size, i, softreset; 1474203123Smav uint8_t *fis = ch->dma.rfis + 0x40; 1475203123Smav uint8_t val; 1476195534Sscottl 1477195534Sscottl /* Get a piece of the workspace for this request */ 1478195534Sscottl ctp = (struct ahci_cmd_tab *) 1479195534Sscottl (ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot)); 1480195534Sscottl /* Setup the FIS for this request */ 1481279918Smav if (!(fis_size = ahci_setup_fis(ch, ctp, ccb, slot->slot))) { 1482195534Sscottl device_printf(ch->dev, "Setting up SATA FIS failed\n"); 1483195534Sscottl ahci_end_transaction(slot, AHCI_ERR_INVALID); 1484195534Sscottl return; 1485195534Sscottl } 1486195534Sscottl /* Setup the command list entry */ 1487195534Sscottl clp = (struct ahci_cmd_list *) 1488195534Sscottl (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot)); 1489214988Smav clp->cmd_flags = htole16( 1490214988Smav (ccb->ccb_h.flags & CAM_DIR_OUT ? AHCI_CMD_WRITE : 0) | 1491214988Smav (ccb->ccb_h.func_code == XPT_SCSI_IO ? 1492214988Smav (AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH) : 0) | 1493214988Smav (fis_size / sizeof(u_int32_t)) | 1494214988Smav (port << 12)); 1495214988Smav clp->prd_length = htole16(slot->dma.nsegs); 1496195534Sscottl /* Special handling for Soft Reset command. */ 1497195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1498203123Smav (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) { 1499203123Smav if (ccb->ataio.cmd.control & ATA_A_RESET) { 1500222304Smav softreset = 1; 1501203123Smav /* Kick controller into sane state */ 1502279918Smav ahci_stop(ch); 1503279918Smav ahci_clo(ch); 1504279918Smav ahci_start(ch, 0); 1505203123Smav clp->cmd_flags |= AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY; 1506203123Smav } else { 1507222304Smav softreset = 2; 1508203123Smav /* Prepare FIS receive area for check. */ 1509203123Smav for (i = 0; i < 20; i++) 1510203123Smav fis[i] = 0xff; 1511203123Smav } 1512222304Smav } else 1513222304Smav softreset = 0; 1514195534Sscottl clp->bytecount = 0; 1515195534Sscottl clp->cmd_table_phys = htole64(ch->dma.work_bus + AHCI_CT_OFFSET + 1516195534Sscottl (AHCI_CT_SIZE * slot->slot)); 1517195534Sscottl bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 1518214988Smav BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1519195534Sscottl bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map, 1520195534Sscottl BUS_DMASYNC_PREREAD); 1521195534Sscottl /* Set ACTIVE bit for NCQ commands. */ 1522195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1523195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1524195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SACT, 1 << slot->slot); 1525195534Sscottl } 1526203123Smav /* If FBS is enabled, set PMP port. */ 1527203123Smav if (ch->fbs_enabled) { 1528203123Smav ATA_OUTL(ch->r_mem, AHCI_P_FBS, AHCI_P_FBS_EN | 1529203123Smav (port << AHCI_P_FBS_DEV_SHIFT)); 1530203123Smav } 1531195534Sscottl /* Issue command to the controller. */ 1532195534Sscottl slot->state = AHCI_SLOT_RUNNING; 1533195534Sscottl ch->rslots |= (1 << slot->slot); 1534195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CI, (1 << slot->slot)); 1535195534Sscottl /* Device reset commands doesn't interrupt. Poll them. */ 1536195534Sscottl if (ccb->ccb_h.func_code == XPT_ATA_IO && 1537222304Smav (ccb->ataio.cmd.command == ATA_DEVICE_RESET || softreset)) { 1538220777Smav int count, timeout = ccb->ccb_h.timeout * 100; 1539195534Sscottl enum ahci_err_type et = AHCI_ERR_NONE; 1540195534Sscottl 1541195534Sscottl for (count = 0; count < timeout; count++) { 1542220777Smav DELAY(10); 1543195534Sscottl if (!(ATA_INL(ch->r_mem, AHCI_P_CI) & (1 << slot->slot))) 1544195534Sscottl break; 1545222304Smav if ((ATA_INL(ch->r_mem, AHCI_P_TFD) & ATA_S_ERROR) && 1546222304Smav softreset != 1) { 1547222285Smav#if 0 1548195534Sscottl device_printf(ch->dev, 1549195534Sscottl "Poll error on slot %d, TFD: %04x\n", 1550195534Sscottl slot->slot, ATA_INL(ch->r_mem, AHCI_P_TFD)); 1551222285Smav#endif 1552195534Sscottl et = AHCI_ERR_TFE; 1553195534Sscottl break; 1554195534Sscottl } 1555198851Smav /* Workaround for ATI SB600/SB700 chipsets. */ 1556198851Smav if (ccb->ccb_h.target_id == 15 && 1557279918Smav (ch->quirks & AHCI_Q_ATI_PMP_BUG) && 1558198851Smav (ATA_INL(ch->r_mem, AHCI_P_IS) & AHCI_P_IX_IPM)) { 1559198851Smav et = AHCI_ERR_TIMEOUT; 1560198851Smav break; 1561198851Smav } 1562195534Sscottl } 1563222304Smav 1564271523Smav /* 1565271523Smav * Marvell HBAs with non-RAID firmware do not wait for 1566271523Smav * readiness after soft reset, so we have to wait here. 1567271523Smav * Marvell RAIDs do not have this problem, but instead 1568271523Smav * sometimes forget to update FIS receive area, breaking 1569271523Smav * this wait. 1570271523Smav */ 1571271523Smav if ((ch->quirks & AHCI_Q_NOBSYRES) == 0 && 1572279918Smav (ch->quirks & AHCI_Q_ATI_PMP_BUG) == 0 && 1573271523Smav softreset == 2 && et == AHCI_ERR_NONE) { 1574222304Smav while ((val = fis[2]) & ATA_S_BUSY) { 1575222304Smav DELAY(10); 1576222304Smav if (count++ >= timeout) 1577222304Smav break; 1578222304Smav } 1579222304Smav } 1580222304Smav 1581195534Sscottl if (timeout && (count >= timeout)) { 1582279918Smav device_printf(ch->dev, "Poll timeout on slot %d port %d\n", 1583222304Smav slot->slot, port); 1584279918Smav device_printf(ch->dev, "is %08x cs %08x ss %08x " 1585224498Smav "rs %08x tfd %02x serr %08x cmd %08x\n", 1586203108Smav ATA_INL(ch->r_mem, AHCI_P_IS), 1587203108Smav ATA_INL(ch->r_mem, AHCI_P_CI), 1588203108Smav ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots, 1589203108Smav ATA_INL(ch->r_mem, AHCI_P_TFD), 1590224498Smav ATA_INL(ch->r_mem, AHCI_P_SERR), 1591224498Smav ATA_INL(ch->r_mem, AHCI_P_CMD)); 1592195534Sscottl et = AHCI_ERR_TIMEOUT; 1593195534Sscottl } 1594222304Smav 1595203123Smav /* Kick controller into sane state and enable FBS. */ 1596222304Smav if (softreset == 2) 1597222285Smav ch->eslots |= (1 << slot->slot); 1598222285Smav ahci_end_transaction(slot, et); 1599195534Sscottl return; 1600195534Sscottl } 1601195534Sscottl /* Start command execution timeout */ 1602275982Ssmh callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout / 2, 1603275982Ssmh 0, (timeout_t*)ahci_timeout, slot, 0); 1604195534Sscottl return; 1605195534Sscottl} 1606195534Sscottl 1607203873Smav/* Must be called with channel locked. */ 1608203873Smavstatic void 1609279918Smavahci_process_timeout(struct ahci_channel *ch) 1610203873Smav{ 1611203873Smav int i; 1612203873Smav 1613203873Smav mtx_assert(&ch->mtx, MA_OWNED); 1614203873Smav /* Handle the rest of commands. */ 1615203873Smav for (i = 0; i < ch->numslots; i++) { 1616203873Smav /* Do we have a running request on slot? */ 1617203873Smav if (ch->slot[i].state < AHCI_SLOT_RUNNING) 1618203873Smav continue; 1619203873Smav ahci_end_transaction(&ch->slot[i], AHCI_ERR_TIMEOUT); 1620203873Smav } 1621203873Smav} 1622203873Smav 1623203873Smav/* Must be called with channel locked. */ 1624203873Smavstatic void 1625279918Smavahci_rearm_timeout(struct ahci_channel *ch) 1626203873Smav{ 1627203873Smav int i; 1628203873Smav 1629203873Smav mtx_assert(&ch->mtx, MA_OWNED); 1630203873Smav for (i = 0; i < ch->numslots; i++) { 1631203873Smav struct ahci_slot *slot = &ch->slot[i]; 1632203873Smav 1633203873Smav /* Do we have a running request on slot? */ 1634203873Smav if (slot->state < AHCI_SLOT_RUNNING) 1635203873Smav continue; 1636203873Smav if ((ch->toslots & (1 << i)) == 0) 1637203873Smav continue; 1638275982Ssmh callout_reset_sbt(&slot->timeout, 1639275982Ssmh SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0, 1640275982Ssmh (timeout_t*)ahci_timeout, slot, 0); 1641203873Smav } 1642203873Smav} 1643203873Smav 1644195534Sscottl/* Locked by callout mechanism. */ 1645195534Sscottlstatic void 1646195534Sscottlahci_timeout(struct ahci_slot *slot) 1647195534Sscottl{ 1648279918Smav struct ahci_channel *ch = slot->ch; 1649279918Smav device_t dev = ch->dev; 1650198319Smav uint32_t sstatus; 1651198319Smav int ccs; 1652195534Sscottl int i; 1653195534Sscottl 1654196656Smav /* Check for stale timeout. */ 1655198319Smav if (slot->state < AHCI_SLOT_RUNNING) 1656196656Smav return; 1657196656Smav 1658198319Smav /* Check if slot was not being executed last time we checked. */ 1659198319Smav if (slot->state < AHCI_SLOT_EXECUTING) { 1660198319Smav /* Check if slot started executing. */ 1661198319Smav sstatus = ATA_INL(ch->r_mem, AHCI_P_SACT); 1662198319Smav ccs = (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CCS_MASK) 1663198319Smav >> AHCI_P_CMD_CCS_SHIFT; 1664203123Smav if ((sstatus & (1 << slot->slot)) != 0 || ccs == slot->slot || 1665224498Smav ch->fbs_enabled || ch->wrongccs) 1666198319Smav slot->state = AHCI_SLOT_EXECUTING; 1667224498Smav else if ((ch->rslots & (1 << ccs)) == 0) { 1668224498Smav ch->wrongccs = 1; 1669224498Smav slot->state = AHCI_SLOT_EXECUTING; 1670224498Smav } 1671198319Smav 1672275982Ssmh callout_reset_sbt(&slot->timeout, 1673275982Ssmh SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0, 1674275982Ssmh (timeout_t*)ahci_timeout, slot, 0); 1675198319Smav return; 1676198319Smav } 1677198319Smav 1678222304Smav device_printf(dev, "Timeout on slot %d port %d\n", 1679222304Smav slot->slot, slot->ccb->ccb_h.target_id & 0x0f); 1680224498Smav device_printf(dev, "is %08x cs %08x ss %08x rs %08x tfd %02x " 1681224498Smav "serr %08x cmd %08x\n", 1682198319Smav ATA_INL(ch->r_mem, AHCI_P_IS), ATA_INL(ch->r_mem, AHCI_P_CI), 1683198319Smav ATA_INL(ch->r_mem, AHCI_P_SACT), ch->rslots, 1684224498Smav ATA_INL(ch->r_mem, AHCI_P_TFD), ATA_INL(ch->r_mem, AHCI_P_SERR), 1685224498Smav ATA_INL(ch->r_mem, AHCI_P_CMD)); 1686195534Sscottl 1687197838Smav /* Handle frozen command. */ 1688195534Sscottl if (ch->frozen) { 1689195534Sscottl union ccb *fccb = ch->frozen; 1690195534Sscottl ch->frozen = NULL; 1691195534Sscottl fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 1692198319Smav if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 1693198319Smav xpt_freeze_devq(fccb->ccb_h.path, 1); 1694198319Smav fccb->ccb_h.status |= CAM_DEV_QFRZN; 1695198319Smav } 1696260387Sscottl ahci_done(ch, fccb); 1697195534Sscottl } 1698224498Smav if (!ch->fbs_enabled && !ch->wrongccs) { 1699203873Smav /* Without FBS we know real timeout source. */ 1700203873Smav ch->fatalerr = 1; 1701203873Smav /* Handle command with timeout. */ 1702203873Smav ahci_end_transaction(&ch->slot[slot->slot], AHCI_ERR_TIMEOUT); 1703203873Smav /* Handle the rest of commands. */ 1704203873Smav for (i = 0; i < ch->numslots; i++) { 1705203873Smav /* Do we have a running request on slot? */ 1706203873Smav if (ch->slot[i].state < AHCI_SLOT_RUNNING) 1707203873Smav continue; 1708203873Smav ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT); 1709203873Smav } 1710203873Smav } else { 1711203873Smav /* With FBS we wait for other commands timeout and pray. */ 1712203873Smav if (ch->toslots == 0) 1713203873Smav xpt_freeze_simq(ch->sim, 1); 1714203873Smav ch->toslots |= (1 << slot->slot); 1715203873Smav if ((ch->rslots & ~ch->toslots) == 0) 1716279918Smav ahci_process_timeout(ch); 1717203873Smav else 1718203873Smav device_printf(dev, " ... waiting for slots %08x\n", 1719203873Smav ch->rslots & ~ch->toslots); 1720195534Sscottl } 1721195534Sscottl} 1722195534Sscottl 1723195534Sscottl/* Must be called with channel locked. */ 1724195534Sscottlstatic void 1725195534Sscottlahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et) 1726195534Sscottl{ 1727279918Smav struct ahci_channel *ch = slot->ch; 1728195534Sscottl union ccb *ccb = slot->ccb; 1729214988Smav struct ahci_cmd_list *clp; 1730212732Smav int lastto; 1731222304Smav uint32_t sig; 1732195534Sscottl 1733195534Sscottl bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 1734214988Smav BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1735214988Smav clp = (struct ahci_cmd_list *) 1736214988Smav (ch->dma.work + AHCI_CL_OFFSET + (AHCI_CL_SIZE * slot->slot)); 1737195534Sscottl /* Read result registers to the result struct 1738195534Sscottl * May be incorrect if several commands finished same time, 1739195534Sscottl * so read only when sure or have to. 1740195534Sscottl */ 1741195534Sscottl if (ccb->ccb_h.func_code == XPT_ATA_IO) { 1742195534Sscottl struct ata_res *res = &ccb->ataio.res; 1743195534Sscottl 1744195534Sscottl if ((et == AHCI_ERR_TFE) || 1745195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) { 1746195534Sscottl u_int8_t *fis = ch->dma.rfis + 0x40; 1747195534Sscottl 1748195534Sscottl bus_dmamap_sync(ch->dma.rfis_tag, ch->dma.rfis_map, 1749195534Sscottl BUS_DMASYNC_POSTREAD); 1750203123Smav if (ch->fbs_enabled) { 1751203123Smav fis += ccb->ccb_h.target_id * 256; 1752203123Smav res->status = fis[2]; 1753203123Smav res->error = fis[3]; 1754203123Smav } else { 1755203123Smav uint16_t tfd = ATA_INL(ch->r_mem, AHCI_P_TFD); 1756203123Smav 1757203123Smav res->status = tfd; 1758203123Smav res->error = tfd >> 8; 1759203123Smav } 1760195534Sscottl res->lba_low = fis[4]; 1761195534Sscottl res->lba_mid = fis[5]; 1762195534Sscottl res->lba_high = fis[6]; 1763195534Sscottl res->device = fis[7]; 1764195534Sscottl res->lba_low_exp = fis[8]; 1765195534Sscottl res->lba_mid_exp = fis[9]; 1766195534Sscottl res->lba_high_exp = fis[10]; 1767195534Sscottl res->sector_count = fis[12]; 1768195534Sscottl res->sector_count_exp = fis[13]; 1769222304Smav 1770222304Smav /* 1771222304Smav * Some weird controllers do not return signature in 1772222304Smav * FIS receive area. Read it from PxSIG register. 1773222304Smav */ 1774222304Smav if ((ch->quirks & AHCI_Q_ALTSIG) && 1775222304Smav (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 1776222304Smav (ccb->ataio.cmd.control & ATA_A_RESET) == 0) { 1777222304Smav sig = ATA_INL(ch->r_mem, AHCI_P_SIG); 1778222304Smav res->lba_high = sig >> 24; 1779222304Smav res->lba_mid = sig >> 16; 1780222304Smav res->lba_low = sig >> 8; 1781222304Smav res->sector_count = sig; 1782222304Smav } 1783195534Sscottl } else 1784195534Sscottl bzero(res, sizeof(*res)); 1785214988Smav if ((ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) == 0 && 1786218596Smav (ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 1787218596Smav (ch->quirks & AHCI_Q_NOCOUNT) == 0) { 1788214988Smav ccb->ataio.resid = 1789214988Smav ccb->ataio.dxfer_len - le32toh(clp->bytecount); 1790214988Smav } 1791214988Smav } else { 1792218596Smav if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 1793218596Smav (ch->quirks & AHCI_Q_NOCOUNT) == 0) { 1794214988Smav ccb->csio.resid = 1795214988Smav ccb->csio.dxfer_len - le32toh(clp->bytecount); 1796214988Smav } 1797195534Sscottl } 1798195534Sscottl if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 1799195534Sscottl bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 1800195534Sscottl (ccb->ccb_h.flags & CAM_DIR_IN) ? 1801195534Sscottl BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1802195534Sscottl bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map); 1803195534Sscottl } 1804203123Smav if (et != AHCI_ERR_NONE) 1805203123Smav ch->eslots |= (1 << slot->slot); 1806198319Smav /* In case of error, freeze device for proper recovery. */ 1807220565Smav if ((et != AHCI_ERR_NONE) && (!ch->recoverycmd) && 1808198319Smav !(ccb->ccb_h.status & CAM_DEV_QFRZN)) { 1809198319Smav xpt_freeze_devq(ccb->ccb_h.path, 1); 1810198319Smav ccb->ccb_h.status |= CAM_DEV_QFRZN; 1811198319Smav } 1812195534Sscottl /* Set proper result status. */ 1813195534Sscottl ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1814195534Sscottl switch (et) { 1815195534Sscottl case AHCI_ERR_NONE: 1816195534Sscottl ccb->ccb_h.status |= CAM_REQ_CMP; 1817195534Sscottl if (ccb->ccb_h.func_code == XPT_SCSI_IO) 1818195534Sscottl ccb->csio.scsi_status = SCSI_STATUS_OK; 1819195534Sscottl break; 1820195534Sscottl case AHCI_ERR_INVALID: 1821198851Smav ch->fatalerr = 1; 1822195534Sscottl ccb->ccb_h.status |= CAM_REQ_INVALID; 1823195534Sscottl break; 1824195534Sscottl case AHCI_ERR_INNOCENT: 1825195534Sscottl ccb->ccb_h.status |= CAM_REQUEUE_REQ; 1826195534Sscottl break; 1827195534Sscottl case AHCI_ERR_TFE: 1828198319Smav case AHCI_ERR_NCQ: 1829195534Sscottl if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 1830195534Sscottl ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR; 1831195534Sscottl ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; 1832195534Sscottl } else { 1833195534Sscottl ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR; 1834195534Sscottl } 1835195534Sscottl break; 1836195534Sscottl case AHCI_ERR_SATA: 1837198851Smav ch->fatalerr = 1; 1838220565Smav if (!ch->recoverycmd) { 1839198319Smav xpt_freeze_simq(ch->sim, 1); 1840198319Smav ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1841198319Smav ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 1842198319Smav } 1843198319Smav ccb->ccb_h.status |= CAM_UNCOR_PARITY; 1844195534Sscottl break; 1845195534Sscottl case AHCI_ERR_TIMEOUT: 1846220565Smav if (!ch->recoverycmd) { 1847198319Smav xpt_freeze_simq(ch->sim, 1); 1848198319Smav ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1849198319Smav ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 1850198319Smav } 1851195534Sscottl ccb->ccb_h.status |= CAM_CMD_TIMEOUT; 1852195534Sscottl break; 1853195534Sscottl default: 1854198851Smav ch->fatalerr = 1; 1855195534Sscottl ccb->ccb_h.status |= CAM_REQ_CMP_ERR; 1856195534Sscottl } 1857195534Sscottl /* Free slot. */ 1858199747Smav ch->oslots &= ~(1 << slot->slot); 1859195534Sscottl ch->rslots &= ~(1 << slot->slot); 1860195534Sscottl ch->aslots &= ~(1 << slot->slot); 1861195534Sscottl slot->state = AHCI_SLOT_EMPTY; 1862195534Sscottl slot->ccb = NULL; 1863195534Sscottl /* Update channel stats. */ 1864195534Sscottl ch->numrslots--; 1865203123Smav ch->numrslotspd[ccb->ccb_h.target_id]--; 1866195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1867195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1868195534Sscottl ch->numtslots--; 1869203123Smav ch->numtslotspd[ccb->ccb_h.target_id]--; 1870195534Sscottl } 1871212732Smav /* Cancel timeout state if request completed normally. */ 1872212732Smav if (et != AHCI_ERR_TIMEOUT) { 1873212732Smav lastto = (ch->toslots == (1 << slot->slot)); 1874212732Smav ch->toslots &= ~(1 << slot->slot); 1875212732Smav if (lastto) 1876212732Smav xpt_release_simq(ch->sim, TRUE); 1877212732Smav } 1878195534Sscottl /* If it was first request of reset sequence and there is no error, 1879195534Sscottl * proceed to second request. */ 1880195534Sscottl if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1881195534Sscottl (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 1882195534Sscottl (ccb->ataio.cmd.control & ATA_A_RESET) && 1883195534Sscottl et == AHCI_ERR_NONE) { 1884195534Sscottl ccb->ataio.cmd.control &= ~ATA_A_RESET; 1885279918Smav ahci_begin_transaction(ch, ccb); 1886195534Sscottl return; 1887195534Sscottl } 1888198851Smav /* If it was our READ LOG command - process it. */ 1889220565Smav if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) { 1890279918Smav ahci_process_read_log(ch, ccb); 1891220565Smav /* If it was our REQUEST SENSE command - process it. */ 1892220565Smav } else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) { 1893279918Smav ahci_process_request_sense(ch, ccb); 1894220565Smav /* If it was NCQ or ATAPI command error, put result on hold. */ 1895220565Smav } else if (et == AHCI_ERR_NCQ || 1896220565Smav ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR && 1897220565Smav (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) { 1898195534Sscottl ch->hold[slot->slot] = ccb; 1899203123Smav ch->numhslots++; 1900198851Smav } else 1901260387Sscottl ahci_done(ch, ccb); 1902198851Smav /* If we have no other active commands, ... */ 1903198851Smav if (ch->rslots == 0) { 1904198851Smav /* if there was fatal error - reset port. */ 1905203873Smav if (ch->toslots != 0 || ch->fatalerr) { 1906279918Smav ahci_reset(ch); 1907203123Smav } else { 1908203123Smav /* if we have slots in error, we can reinit port. */ 1909203123Smav if (ch->eslots != 0) { 1910279918Smav ahci_stop(ch); 1911279918Smav ahci_clo(ch); 1912279918Smav ahci_start(ch, 1); 1913203123Smav } 1914203123Smav /* if there commands on hold, we can do READ LOG. */ 1915220565Smav if (!ch->recoverycmd && ch->numhslots) 1916279918Smav ahci_issue_recovery(ch); 1917198851Smav } 1918203873Smav /* If all the rest of commands are in timeout - give them chance. */ 1919203873Smav } else if ((ch->rslots & ~ch->toslots) == 0 && 1920203873Smav et != AHCI_ERR_TIMEOUT) 1921279918Smav ahci_rearm_timeout(ch); 1922222285Smav /* Unfreeze frozen command. */ 1923279918Smav if (ch->frozen && !ahci_check_collision(ch, ch->frozen)) { 1924222285Smav union ccb *fccb = ch->frozen; 1925222285Smav ch->frozen = NULL; 1926279918Smav ahci_begin_transaction(ch, fccb); 1927222285Smav xpt_release_simq(ch->sim, TRUE); 1928222285Smav } 1929196656Smav /* Start PM timer. */ 1930207499Smav if (ch->numrslots == 0 && ch->pm_level > 3 && 1931207499Smav (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) { 1932196656Smav callout_schedule(&ch->pm_timer, 1933196656Smav (ch->pm_level == 4) ? hz / 1000 : hz / 8); 1934196656Smav } 1935195534Sscottl} 1936195534Sscottl 1937195534Sscottlstatic void 1938279918Smavahci_issue_recovery(struct ahci_channel *ch) 1939195534Sscottl{ 1940195534Sscottl union ccb *ccb; 1941195534Sscottl struct ccb_ataio *ataio; 1942220565Smav struct ccb_scsiio *csio; 1943195534Sscottl int i; 1944195534Sscottl 1945220830Smav /* Find some held command. */ 1946195534Sscottl for (i = 0; i < ch->numslots; i++) { 1947195534Sscottl if (ch->hold[i]) 1948195534Sscottl break; 1949195534Sscottl } 1950195534Sscottl ccb = xpt_alloc_ccb_nowait(); 1951195534Sscottl if (ccb == NULL) { 1952279918Smav device_printf(ch->dev, "Unable to allocate recovery command\n"); 1953220822Smavcompleteall: 1954220830Smav /* We can't do anything -- complete held commands. */ 1955220822Smav for (i = 0; i < ch->numslots; i++) { 1956220822Smav if (ch->hold[i] == NULL) 1957220822Smav continue; 1958220822Smav ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 1959220822Smav ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL; 1960260387Sscottl ahci_done(ch, ch->hold[i]); 1961220822Smav ch->hold[i] = NULL; 1962220822Smav ch->numhslots--; 1963220822Smav } 1964279918Smav ahci_reset(ch); 1965220822Smav return; 1966195534Sscottl } 1967195534Sscottl ccb->ccb_h = ch->hold[i]->ccb_h; /* Reuse old header. */ 1968220565Smav if (ccb->ccb_h.func_code == XPT_ATA_IO) { 1969220565Smav /* READ LOG */ 1970220565Smav ccb->ccb_h.recovery_type = RECOVERY_READ_LOG; 1971220565Smav ccb->ccb_h.func_code = XPT_ATA_IO; 1972220565Smav ccb->ccb_h.flags = CAM_DIR_IN; 1973220565Smav ccb->ccb_h.timeout = 1000; /* 1s should be enough. */ 1974220565Smav ataio = &ccb->ataio; 1975220565Smav ataio->data_ptr = malloc(512, M_AHCI, M_NOWAIT); 1976220565Smav if (ataio->data_ptr == NULL) { 1977220565Smav xpt_free_ccb(ccb); 1978279918Smav device_printf(ch->dev, 1979220830Smav "Unable to allocate memory for READ LOG command\n"); 1980220822Smav goto completeall; 1981220565Smav } 1982220565Smav ataio->dxfer_len = 512; 1983220565Smav bzero(&ataio->cmd, sizeof(ataio->cmd)); 1984220565Smav ataio->cmd.flags = CAM_ATAIO_48BIT; 1985220565Smav ataio->cmd.command = 0x2F; /* READ LOG EXT */ 1986220565Smav ataio->cmd.sector_count = 1; 1987220565Smav ataio->cmd.sector_count_exp = 0; 1988220565Smav ataio->cmd.lba_low = 0x10; 1989220565Smav ataio->cmd.lba_mid = 0; 1990220565Smav ataio->cmd.lba_mid_exp = 0; 1991220565Smav } else { 1992220565Smav /* REQUEST SENSE */ 1993220565Smav ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE; 1994220565Smav ccb->ccb_h.recovery_slot = i; 1995220565Smav ccb->ccb_h.func_code = XPT_SCSI_IO; 1996220565Smav ccb->ccb_h.flags = CAM_DIR_IN; 1997220565Smav ccb->ccb_h.status = 0; 1998220565Smav ccb->ccb_h.timeout = 1000; /* 1s should be enough. */ 1999220565Smav csio = &ccb->csio; 2000220565Smav csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data; 2001220565Smav csio->dxfer_len = ch->hold[i]->csio.sense_len; 2002220565Smav csio->cdb_len = 6; 2003220565Smav bzero(&csio->cdb_io, sizeof(csio->cdb_io)); 2004220565Smav csio->cdb_io.cdb_bytes[0] = 0x03; 2005220565Smav csio->cdb_io.cdb_bytes[4] = csio->dxfer_len; 2006195534Sscottl } 2007220565Smav /* Freeze SIM while doing recovery. */ 2008220822Smav ch->recoverycmd = 1; 2009198319Smav xpt_freeze_simq(ch->sim, 1); 2010279918Smav ahci_begin_transaction(ch, ccb); 2011195534Sscottl} 2012195534Sscottl 2013195534Sscottlstatic void 2014279918Smavahci_process_read_log(struct ahci_channel *ch, union ccb *ccb) 2015195534Sscottl{ 2016195534Sscottl uint8_t *data; 2017195534Sscottl struct ata_res *res; 2018195534Sscottl int i; 2019195534Sscottl 2020220565Smav ch->recoverycmd = 0; 2021195534Sscottl 2022195534Sscottl data = ccb->ataio.data_ptr; 2023195534Sscottl if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP && 2024195534Sscottl (data[0] & 0x80) == 0) { 2025195534Sscottl for (i = 0; i < ch->numslots; i++) { 2026195534Sscottl if (!ch->hold[i]) 2027195534Sscottl continue; 2028220565Smav if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO) 2029220565Smav continue; 2030195534Sscottl if ((data[0] & 0x1F) == i) { 2031195534Sscottl res = &ch->hold[i]->ataio.res; 2032195534Sscottl res->status = data[2]; 2033195534Sscottl res->error = data[3]; 2034195534Sscottl res->lba_low = data[4]; 2035195534Sscottl res->lba_mid = data[5]; 2036195534Sscottl res->lba_high = data[6]; 2037195534Sscottl res->device = data[7]; 2038195534Sscottl res->lba_low_exp = data[8]; 2039195534Sscottl res->lba_mid_exp = data[9]; 2040195534Sscottl res->lba_high_exp = data[10]; 2041195534Sscottl res->sector_count = data[12]; 2042195534Sscottl res->sector_count_exp = data[13]; 2043195534Sscottl } else { 2044195534Sscottl ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 2045195534Sscottl ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ; 2046195534Sscottl } 2047260387Sscottl ahci_done(ch, ch->hold[i]); 2048195534Sscottl ch->hold[i] = NULL; 2049203123Smav ch->numhslots--; 2050195534Sscottl } 2051195534Sscottl } else { 2052195534Sscottl if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) 2053279918Smav device_printf(ch->dev, "Error while READ LOG EXT\n"); 2054195534Sscottl else if ((data[0] & 0x80) == 0) { 2055279918Smav device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n"); 2056195534Sscottl } 2057195534Sscottl for (i = 0; i < ch->numslots; i++) { 2058195534Sscottl if (!ch->hold[i]) 2059195534Sscottl continue; 2060220565Smav if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO) 2061220565Smav continue; 2062260387Sscottl ahci_done(ch, ch->hold[i]); 2063195534Sscottl ch->hold[i] = NULL; 2064203123Smav ch->numhslots--; 2065195534Sscottl } 2066195534Sscottl } 2067195534Sscottl free(ccb->ataio.data_ptr, M_AHCI); 2068195534Sscottl xpt_free_ccb(ccb); 2069198319Smav xpt_release_simq(ch->sim, TRUE); 2070195534Sscottl} 2071195534Sscottl 2072195534Sscottlstatic void 2073279918Smavahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb) 2074220565Smav{ 2075220565Smav int i; 2076220565Smav 2077220565Smav ch->recoverycmd = 0; 2078220565Smav 2079220565Smav i = ccb->ccb_h.recovery_slot; 2080220565Smav if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { 2081220565Smav ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID; 2082220565Smav } else { 2083220565Smav ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 2084220565Smav ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL; 2085220565Smav } 2086260387Sscottl ahci_done(ch, ch->hold[i]); 2087220565Smav ch->hold[i] = NULL; 2088220565Smav ch->numhslots--; 2089220565Smav xpt_free_ccb(ccb); 2090220565Smav xpt_release_simq(ch->sim, TRUE); 2091220565Smav} 2092220565Smav 2093220565Smavstatic void 2094279918Smavahci_start(struct ahci_channel *ch, int fbs) 2095195534Sscottl{ 2096195534Sscottl u_int32_t cmd; 2097195534Sscottl 2098195534Sscottl /* Clear SATA error register */ 2099195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xFFFFFFFF); 2100195534Sscottl /* Clear any interrupts pending on this channel */ 2101195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IS, 0xFFFFFFFF); 2102203123Smav /* Configure FIS-based switching if supported. */ 2103203123Smav if (ch->chcaps & AHCI_P_CMD_FBSCP) { 2104203123Smav ch->fbs_enabled = (fbs && ch->pm_present) ? 1 : 0; 2105203123Smav ATA_OUTL(ch->r_mem, AHCI_P_FBS, 2106203123Smav ch->fbs_enabled ? AHCI_P_FBS_EN : 0); 2107203123Smav } 2108195534Sscottl /* Start operations on this channel */ 2109195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2110207430Smav cmd &= ~AHCI_P_CMD_PMA; 2111195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_ST | 2112195534Sscottl (ch->pm_present ? AHCI_P_CMD_PMA : 0)); 2113195534Sscottl} 2114195534Sscottl 2115195534Sscottlstatic void 2116279918Smavahci_stop(struct ahci_channel *ch) 2117195534Sscottl{ 2118195534Sscottl u_int32_t cmd; 2119195534Sscottl int timeout; 2120195534Sscottl 2121195534Sscottl /* Kill all activity on this channel */ 2122195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2123195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_ST); 2124195534Sscottl /* Wait for activity stop. */ 2125195534Sscottl timeout = 0; 2126195534Sscottl do { 2127220777Smav DELAY(10); 2128220777Smav if (timeout++ > 50000) { 2129279918Smav device_printf(ch->dev, "stopping AHCI engine failed\n"); 2130195534Sscottl break; 2131195534Sscottl } 2132195534Sscottl } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CR); 2133203123Smav ch->eslots = 0; 2134195534Sscottl} 2135195534Sscottl 2136195534Sscottlstatic void 2137279918Smavahci_clo(struct ahci_channel *ch) 2138195534Sscottl{ 2139195534Sscottl u_int32_t cmd; 2140195534Sscottl int timeout; 2141195534Sscottl 2142195534Sscottl /* Issue Command List Override if supported */ 2143195534Sscottl if (ch->caps & AHCI_CAP_SCLO) { 2144195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2145195534Sscottl cmd |= AHCI_P_CMD_CLO; 2146195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd); 2147195534Sscottl timeout = 0; 2148195534Sscottl do { 2149220777Smav DELAY(10); 2150220777Smav if (timeout++ > 50000) { 2151279918Smav device_printf(ch->dev, "executing CLO failed\n"); 2152195534Sscottl break; 2153195534Sscottl } 2154195534Sscottl } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CLO); 2155195534Sscottl } 2156195534Sscottl} 2157195534Sscottl 2158195534Sscottlstatic void 2159279918Smavahci_stop_fr(struct ahci_channel *ch) 2160195534Sscottl{ 2161195534Sscottl u_int32_t cmd; 2162195534Sscottl int timeout; 2163195534Sscottl 2164195534Sscottl /* Kill all FIS reception on this channel */ 2165195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2166195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd & ~AHCI_P_CMD_FRE); 2167195534Sscottl /* Wait for FIS reception stop. */ 2168195534Sscottl timeout = 0; 2169195534Sscottl do { 2170220777Smav DELAY(10); 2171220777Smav if (timeout++ > 50000) { 2172279918Smav device_printf(ch->dev, "stopping AHCI FR engine failed\n"); 2173195534Sscottl break; 2174195534Sscottl } 2175195534Sscottl } while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_FR); 2176195534Sscottl} 2177195534Sscottl 2178195534Sscottlstatic void 2179279918Smavahci_start_fr(struct ahci_channel *ch) 2180195534Sscottl{ 2181195534Sscottl u_int32_t cmd; 2182195534Sscottl 2183195534Sscottl /* Start FIS reception on this channel */ 2184195534Sscottl cmd = ATA_INL(ch->r_mem, AHCI_P_CMD); 2185195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_CMD, cmd | AHCI_P_CMD_FRE); 2186195534Sscottl} 2187195534Sscottl 2188195534Sscottlstatic int 2189279918Smavahci_wait_ready(struct ahci_channel *ch, int t, int t0) 2190195534Sscottl{ 2191195534Sscottl int timeout = 0; 2192195534Sscottl uint32_t val; 2193195534Sscottl 2194195534Sscottl while ((val = ATA_INL(ch->r_mem, AHCI_P_TFD)) & 2195195534Sscottl (ATA_S_BUSY | ATA_S_DRQ)) { 2196220576Smav if (timeout > t) { 2197220576Smav if (t != 0) { 2198279918Smav device_printf(ch->dev, 2199220576Smav "AHCI reset: device not ready after %dms " 2200220576Smav "(tfd = %08x)\n", 2201220576Smav MAX(t, 0) + t0, val); 2202220576Smav } 2203195534Sscottl return (EBUSY); 2204195534Sscottl } 2205220576Smav DELAY(1000); 2206220576Smav timeout++; 2207220576Smav } 2208195534Sscottl if (bootverbose) 2209279918Smav device_printf(ch->dev, "AHCI reset: device ready after %dms\n", 2210220576Smav timeout + t0); 2211195534Sscottl return (0); 2212195534Sscottl} 2213195534Sscottl 2214195534Sscottlstatic void 2215220576Smavahci_reset_to(void *arg) 2216220576Smav{ 2217279918Smav struct ahci_channel *ch = arg; 2218220576Smav 2219220576Smav if (ch->resetting == 0) 2220220576Smav return; 2221220576Smav ch->resetting--; 2222279918Smav if (ahci_wait_ready(ch, ch->resetting == 0 ? -1 : 0, 2223220576Smav (310 - ch->resetting) * 100) == 0) { 2224220576Smav ch->resetting = 0; 2225279918Smav ahci_start(ch, 1); 2226220576Smav xpt_release_simq(ch->sim, TRUE); 2227220576Smav return; 2228220576Smav } 2229220576Smav if (ch->resetting == 0) { 2230279918Smav ahci_clo(ch); 2231279918Smav ahci_start(ch, 1); 2232220576Smav xpt_release_simq(ch->sim, TRUE); 2233220576Smav return; 2234220576Smav } 2235220576Smav callout_schedule(&ch->reset_timer, hz / 10); 2236220576Smav} 2237220576Smav 2238220576Smavstatic void 2239279918Smavahci_reset(struct ahci_channel *ch) 2240195534Sscottl{ 2241279918Smav struct ahci_controller *ctlr = device_get_softc(device_get_parent(ch->dev)); 2242195534Sscottl int i; 2243195534Sscottl 2244203108Smav xpt_freeze_simq(ch->sim, 1); 2245195534Sscottl if (bootverbose) 2246279918Smav device_printf(ch->dev, "AHCI reset...\n"); 2247220576Smav /* Forget about previous reset. */ 2248220576Smav if (ch->resetting) { 2249220576Smav ch->resetting = 0; 2250220576Smav callout_stop(&ch->reset_timer); 2251220576Smav xpt_release_simq(ch->sim, TRUE); 2252220576Smav } 2253195534Sscottl /* Requeue freezed command. */ 2254195534Sscottl if (ch->frozen) { 2255195534Sscottl union ccb *fccb = ch->frozen; 2256195534Sscottl ch->frozen = NULL; 2257195534Sscottl fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 2258198319Smav if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 2259198319Smav xpt_freeze_devq(fccb->ccb_h.path, 1); 2260198319Smav fccb->ccb_h.status |= CAM_DEV_QFRZN; 2261198319Smav } 2262260387Sscottl ahci_done(ch, fccb); 2263195534Sscottl } 2264195534Sscottl /* Kill the engine and requeue all running commands. */ 2265279918Smav ahci_stop(ch); 2266195534Sscottl for (i = 0; i < ch->numslots; i++) { 2267195534Sscottl /* Do we have a running request on slot? */ 2268195534Sscottl if (ch->slot[i].state < AHCI_SLOT_RUNNING) 2269195534Sscottl continue; 2270195534Sscottl /* XXX; Commands in loading state. */ 2271195534Sscottl ahci_end_transaction(&ch->slot[i], AHCI_ERR_INNOCENT); 2272195534Sscottl } 2273198851Smav for (i = 0; i < ch->numslots; i++) { 2274198851Smav if (!ch->hold[i]) 2275198851Smav continue; 2276260387Sscottl ahci_done(ch, ch->hold[i]); 2277198851Smav ch->hold[i] = NULL; 2278203123Smav ch->numhslots--; 2279198851Smav } 2280203873Smav if (ch->toslots != 0) 2281203873Smav xpt_release_simq(ch->sim, TRUE); 2282203123Smav ch->eslots = 0; 2283203873Smav ch->toslots = 0; 2284224498Smav ch->wrongccs = 0; 2285198851Smav ch->fatalerr = 0; 2286198319Smav /* Tell the XPT about the event */ 2287198319Smav xpt_async(AC_BUS_RESET, ch->path, NULL); 2288195534Sscottl /* Disable port interrupts */ 2289195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 0); 2290195534Sscottl /* Reset and reconnect PHY, */ 2291279918Smav if (!ahci_sata_phy_reset(ch)) { 2292195534Sscottl if (bootverbose) 2293279918Smav device_printf(ch->dev, 2294220576Smav "AHCI reset: device not found\n"); 2295195534Sscottl ch->devices = 0; 2296195534Sscottl /* Enable wanted port interrupts */ 2297195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 2298220657Smav (((ch->pm_level != 0) ? AHCI_P_IX_CPD | AHCI_P_IX_MP : 0) | 2299220657Smav AHCI_P_IX_PRC | AHCI_P_IX_PC)); 2300203108Smav xpt_release_simq(ch->sim, TRUE); 2301195534Sscottl return; 2302195534Sscottl } 2303220576Smav if (bootverbose) 2304279918Smav device_printf(ch->dev, "AHCI reset: device found\n"); 2305195534Sscottl /* Wait for clearing busy status. */ 2306279918Smav if (ahci_wait_ready(ch, dumping ? 31000 : 0, 0)) { 2307220576Smav if (dumping) 2308279918Smav ahci_clo(ch); 2309220576Smav else 2310220576Smav ch->resetting = 310; 2311220576Smav } 2312195534Sscottl ch->devices = 1; 2313195534Sscottl /* Enable wanted port interrupts */ 2314195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_IE, 2315220657Smav (((ch->pm_level != 0) ? AHCI_P_IX_CPD | AHCI_P_IX_MP : 0) | 2316220657Smav AHCI_P_IX_TFE | AHCI_P_IX_HBF | 2317195534Sscottl AHCI_P_IX_HBD | AHCI_P_IX_IF | AHCI_P_IX_OF | 2318220657Smav ((ch->pm_level == 0) ? AHCI_P_IX_PRC : 0) | AHCI_P_IX_PC | 2319196656Smav AHCI_P_IX_DP | AHCI_P_IX_UF | (ctlr->ccc ? 0 : AHCI_P_IX_SDB) | 2320196656Smav AHCI_P_IX_DS | AHCI_P_IX_PS | (ctlr->ccc ? 0 : AHCI_P_IX_DHR))); 2321220576Smav if (ch->resetting) 2322279918Smav callout_reset(&ch->reset_timer, hz / 10, ahci_reset_to, ch); 2323220777Smav else { 2324279918Smav ahci_start(ch, 1); 2325220576Smav xpt_release_simq(ch->sim, TRUE); 2326220777Smav } 2327195534Sscottl} 2328195534Sscottl 2329195534Sscottlstatic int 2330279918Smavahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag) 2331195534Sscottl{ 2332195534Sscottl u_int8_t *fis = &ctp->cfis[0]; 2333195534Sscottl 2334279917Smav bzero(fis, 20); 2335195534Sscottl fis[0] = 0x27; /* host to device */ 2336195534Sscottl fis[1] = (ccb->ccb_h.target_id & 0x0f); 2337195534Sscottl if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 2338195534Sscottl fis[1] |= 0x80; 2339195534Sscottl fis[2] = ATA_PACKET_CMD; 2340199821Smav if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 2341199821Smav ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA) 2342195534Sscottl fis[3] = ATA_F_DMA; 2343195534Sscottl else { 2344195534Sscottl fis[5] = ccb->csio.dxfer_len; 2345195534Sscottl fis[6] = ccb->csio.dxfer_len >> 8; 2346195534Sscottl } 2347195534Sscottl fis[7] = ATA_D_LBA; 2348195534Sscottl fis[15] = ATA_A_4BIT; 2349195534Sscottl bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ? 2350195534Sscottl ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes, 2351195534Sscottl ctp->acmd, ccb->csio.cdb_len); 2352248687Smav bzero(ctp->acmd + ccb->csio.cdb_len, 32 - ccb->csio.cdb_len); 2353195534Sscottl } else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) { 2354195534Sscottl fis[1] |= 0x80; 2355195534Sscottl fis[2] = ccb->ataio.cmd.command; 2356195534Sscottl fis[3] = ccb->ataio.cmd.features; 2357195534Sscottl fis[4] = ccb->ataio.cmd.lba_low; 2358195534Sscottl fis[5] = ccb->ataio.cmd.lba_mid; 2359195534Sscottl fis[6] = ccb->ataio.cmd.lba_high; 2360195534Sscottl fis[7] = ccb->ataio.cmd.device; 2361195534Sscottl fis[8] = ccb->ataio.cmd.lba_low_exp; 2362195534Sscottl fis[9] = ccb->ataio.cmd.lba_mid_exp; 2363195534Sscottl fis[10] = ccb->ataio.cmd.lba_high_exp; 2364195534Sscottl fis[11] = ccb->ataio.cmd.features_exp; 2365195534Sscottl if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) { 2366195534Sscottl fis[12] = tag << 3; 2367195534Sscottl fis[13] = 0; 2368195534Sscottl } else { 2369195534Sscottl fis[12] = ccb->ataio.cmd.sector_count; 2370195534Sscottl fis[13] = ccb->ataio.cmd.sector_count_exp; 2371195534Sscottl } 2372195534Sscottl fis[15] = ATA_A_4BIT; 2373195534Sscottl } else { 2374195534Sscottl fis[15] = ccb->ataio.cmd.control; 2375195534Sscottl } 2376195534Sscottl return (20); 2377195534Sscottl} 2378195534Sscottl 2379195534Sscottlstatic int 2380195534Sscottlahci_sata_connect(struct ahci_channel *ch) 2381195534Sscottl{ 2382195534Sscottl u_int32_t status; 2383220829Smav int timeout, found = 0; 2384195534Sscottl 2385195534Sscottl /* Wait up to 100ms for "connect well" */ 2386220777Smav for (timeout = 0; timeout < 1000 ; timeout++) { 2387195534Sscottl status = ATA_INL(ch->r_mem, AHCI_P_SSTS); 2388220829Smav if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE) 2389220829Smav found = 1; 2390195534Sscottl if (((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_ONLINE) && 2391195534Sscottl ((status & ATA_SS_SPD_MASK) != ATA_SS_SPD_NO_SPEED) && 2392195534Sscottl ((status & ATA_SS_IPM_MASK) == ATA_SS_IPM_ACTIVE)) 2393195534Sscottl break; 2394196656Smav if ((status & ATA_SS_DET_MASK) == ATA_SS_DET_PHY_OFFLINE) { 2395196656Smav if (bootverbose) { 2396196656Smav device_printf(ch->dev, "SATA offline status=%08x\n", 2397196656Smav status); 2398196656Smav } 2399196656Smav return (0); 2400196656Smav } 2401220829Smav if (found == 0 && timeout >= 100) 2402220829Smav break; 2403220777Smav DELAY(100); 2404195534Sscottl } 2405220829Smav if (timeout >= 1000 || !found) { 2406195534Sscottl if (bootverbose) { 2407220829Smav device_printf(ch->dev, 2408220829Smav "SATA connect timeout time=%dus status=%08x\n", 2409220829Smav timeout * 100, status); 2410195534Sscottl } 2411195534Sscottl return (0); 2412195534Sscottl } 2413195534Sscottl if (bootverbose) { 2414220777Smav device_printf(ch->dev, "SATA connect time=%dus status=%08x\n", 2415220777Smav timeout * 100, status); 2416195534Sscottl } 2417195534Sscottl /* Clear SATA error register */ 2418195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SERR, 0xffffffff); 2419195534Sscottl return (1); 2420195534Sscottl} 2421195534Sscottl 2422195534Sscottlstatic int 2423279918Smavahci_sata_phy_reset(struct ahci_channel *ch) 2424195534Sscottl{ 2425199821Smav int sata_rev; 2426195534Sscottl uint32_t val; 2427195534Sscottl 2428220657Smav if (ch->listening) { 2429220657Smav val = ATA_INL(ch->r_mem, AHCI_P_CMD); 2430220657Smav val |= AHCI_P_CMD_SUD; 2431220657Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, val); 2432220657Smav ch->listening = 0; 2433220657Smav } 2434199821Smav sata_rev = ch->user[ch->pm_present ? 15 : 0].revision; 2435199821Smav if (sata_rev == 1) 2436195534Sscottl val = ATA_SC_SPD_SPEED_GEN1; 2437199821Smav else if (sata_rev == 2) 2438195534Sscottl val = ATA_SC_SPD_SPEED_GEN2; 2439199821Smav else if (sata_rev == 3) 2440195534Sscottl val = ATA_SC_SPD_SPEED_GEN3; 2441195534Sscottl else 2442195534Sscottl val = 0; 2443195534Sscottl ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 2444196656Smav ATA_SC_DET_RESET | val | 2445196656Smav ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER); 2446220777Smav DELAY(1000); 2447196656Smav ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 2448195534Sscottl ATA_SC_DET_IDLE | val | ((ch->pm_level > 0) ? 0 : 2449195534Sscottl (ATA_SC_IPM_DIS_PARTIAL | ATA_SC_IPM_DIS_SLUMBER))); 2450203426Smav if (!ahci_sata_connect(ch)) { 2451220657Smav if (ch->caps & AHCI_CAP_SSS) { 2452220657Smav val = ATA_INL(ch->r_mem, AHCI_P_CMD); 2453220657Smav val &= ~AHCI_P_CMD_SUD; 2454220657Smav ATA_OUTL(ch->r_mem, AHCI_P_CMD, val); 2455220657Smav ch->listening = 1; 2456220657Smav } else if (ch->pm_level > 0) 2457203426Smav ATA_OUTL(ch->r_mem, AHCI_P_SCTL, ATA_SC_DET_DISABLE); 2458203426Smav return (0); 2459203426Smav } 2460203426Smav return (1); 2461195534Sscottl} 2462195534Sscottl 2463207430Smavstatic int 2464279918Smavahci_check_ids(struct ahci_channel *ch, union ccb *ccb) 2465207430Smav{ 2466207430Smav 2467207430Smav if (ccb->ccb_h.target_id > ((ch->caps & AHCI_CAP_SPM) ? 15 : 0)) { 2468207430Smav ccb->ccb_h.status = CAM_TID_INVALID; 2469260387Sscottl ahci_done(ch, ccb); 2470207430Smav return (-1); 2471207430Smav } 2472207430Smav if (ccb->ccb_h.target_lun != 0) { 2473207430Smav ccb->ccb_h.status = CAM_LUN_INVALID; 2474260387Sscottl ahci_done(ch, ccb); 2475207430Smav return (-1); 2476207430Smav } 2477207430Smav return (0); 2478207430Smav} 2479207430Smav 2480195534Sscottlstatic void 2481195534Sscottlahciaction(struct cam_sim *sim, union ccb *ccb) 2482195534Sscottl{ 2483195534Sscottl struct ahci_channel *ch; 2484195534Sscottl 2485195534Sscottl CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("ahciaction func_code=%x\n", 2486195534Sscottl ccb->ccb_h.func_code)); 2487195534Sscottl 2488195534Sscottl ch = (struct ahci_channel *)cam_sim_softc(sim); 2489195534Sscottl switch (ccb->ccb_h.func_code) { 2490195534Sscottl /* Common cases first */ 2491195534Sscottl case XPT_ATA_IO: /* Execute the requested I/O operation */ 2492195534Sscottl case XPT_SCSI_IO: 2493279918Smav if (ahci_check_ids(ch, ccb)) 2494207430Smav return; 2495207430Smav if (ch->devices == 0 || 2496207430Smav (ch->pm_present == 0 && 2497207430Smav ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) { 2498195534Sscottl ccb->ccb_h.status = CAM_SEL_TIMEOUT; 2499195534Sscottl break; 2500195534Sscottl } 2501220565Smav ccb->ccb_h.recovery_type = RECOVERY_NONE; 2502195534Sscottl /* Check for command collision. */ 2503279918Smav if (ahci_check_collision(ch, ccb)) { 2504195534Sscottl /* Freeze command. */ 2505195534Sscottl ch->frozen = ccb; 2506195534Sscottl /* We have only one frozen slot, so freeze simq also. */ 2507195534Sscottl xpt_freeze_simq(ch->sim, 1); 2508195534Sscottl return; 2509195534Sscottl } 2510279918Smav ahci_begin_transaction(ch, ccb); 2511207430Smav return; 2512195534Sscottl case XPT_EN_LUN: /* Enable LUN as a target */ 2513195534Sscottl case XPT_TARGET_IO: /* Execute target I/O request */ 2514195534Sscottl case XPT_ACCEPT_TARGET_IO: /* Accept Host Target Mode CDB */ 2515195534Sscottl case XPT_CONT_TARGET_IO: /* Continue Host Target I/O Connection*/ 2516195534Sscottl case XPT_ABORT: /* Abort the specified CCB */ 2517195534Sscottl /* XXX Implement */ 2518195534Sscottl ccb->ccb_h.status = CAM_REQ_INVALID; 2519195534Sscottl break; 2520195534Sscottl case XPT_SET_TRAN_SETTINGS: 2521195534Sscottl { 2522195534Sscottl struct ccb_trans_settings *cts = &ccb->cts; 2523199747Smav struct ahci_device *d; 2524195534Sscottl 2525279918Smav if (ahci_check_ids(ch, ccb)) 2526207430Smav return; 2527199747Smav if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 2528199747Smav d = &ch->curr[ccb->ccb_h.target_id]; 2529199747Smav else 2530199747Smav d = &ch->user[ccb->ccb_h.target_id]; 2531199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION) 2532199747Smav d->revision = cts->xport_specific.sata.revision; 2533199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE) 2534199747Smav d->mode = cts->xport_specific.sata.mode; 2535199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) 2536199747Smav d->bytecount = min(8192, cts->xport_specific.sata.bytecount); 2537199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS) 2538199747Smav d->tags = min(ch->numslots, cts->xport_specific.sata.tags); 2539199747Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM) 2540195534Sscottl ch->pm_present = cts->xport_specific.sata.pm_present; 2541203376Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI) 2542203376Smav d->atapi = cts->xport_specific.sata.atapi; 2543207499Smav if (cts->xport_specific.sata.valid & CTS_SATA_VALID_CAPS) 2544207499Smav d->caps = cts->xport_specific.sata.caps; 2545195534Sscottl ccb->ccb_h.status = CAM_REQ_CMP; 2546195534Sscottl break; 2547195534Sscottl } 2548195534Sscottl case XPT_GET_TRAN_SETTINGS: 2549195534Sscottl /* Get default/user set transfer settings for the target */ 2550195534Sscottl { 2551195534Sscottl struct ccb_trans_settings *cts = &ccb->cts; 2552199747Smav struct ahci_device *d; 2553195534Sscottl uint32_t status; 2554195534Sscottl 2555279918Smav if (ahci_check_ids(ch, ccb)) 2556207430Smav return; 2557199747Smav if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 2558199747Smav d = &ch->curr[ccb->ccb_h.target_id]; 2559199747Smav else 2560199747Smav d = &ch->user[ccb->ccb_h.target_id]; 2561236666Smav cts->protocol = PROTO_UNSPECIFIED; 2562196656Smav cts->protocol_version = PROTO_VERSION_UNSPECIFIED; 2563195534Sscottl cts->transport = XPORT_SATA; 2564196656Smav cts->transport_version = XPORT_VERSION_UNSPECIFIED; 2565195534Sscottl cts->proto_specific.valid = 0; 2566195534Sscottl cts->xport_specific.sata.valid = 0; 2567199747Smav if (cts->type == CTS_TYPE_CURRENT_SETTINGS && 2568199747Smav (ccb->ccb_h.target_id == 15 || 2569199747Smav (ccb->ccb_h.target_id == 0 && !ch->pm_present))) { 2570195534Sscottl status = ATA_INL(ch->r_mem, AHCI_P_SSTS) & ATA_SS_SPD_MASK; 2571199747Smav if (status & 0x0f0) { 2572199747Smav cts->xport_specific.sata.revision = 2573199747Smav (status & 0x0f0) >> 4; 2574199747Smav cts->xport_specific.sata.valid |= 2575199747Smav CTS_SATA_VALID_REVISION; 2576199747Smav } 2577207499Smav cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D; 2578207499Smav if (ch->pm_level) { 2579207499Smav if (ch->caps & (AHCI_CAP_PSC | AHCI_CAP_SSC)) 2580207499Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ; 2581207499Smav if (ch->caps2 & AHCI_CAP2_APST) 2582207499Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_APST; 2583207499Smav } 2584207499Smav if ((ch->caps & AHCI_CAP_SNCQ) && 2585207499Smav (ch->quirks & AHCI_Q_NOAA) == 0) 2586207499Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_DMAAA; 2587220602Smav cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN; 2588207499Smav cts->xport_specific.sata.caps &= 2589207499Smav ch->user[ccb->ccb_h.target_id].caps; 2590207499Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 2591195534Sscottl } else { 2592199747Smav cts->xport_specific.sata.revision = d->revision; 2593199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION; 2594207499Smav cts->xport_specific.sata.caps = d->caps; 2595207499Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 2596195534Sscottl } 2597199747Smav cts->xport_specific.sata.mode = d->mode; 2598199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE; 2599199747Smav cts->xport_specific.sata.bytecount = d->bytecount; 2600199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT; 2601199747Smav cts->xport_specific.sata.pm_present = ch->pm_present; 2602195534Sscottl cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM; 2603199747Smav cts->xport_specific.sata.tags = d->tags; 2604199747Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS; 2605203376Smav cts->xport_specific.sata.atapi = d->atapi; 2606203376Smav cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI; 2607195534Sscottl ccb->ccb_h.status = CAM_REQ_CMP; 2608195534Sscottl break; 2609195534Sscottl } 2610195534Sscottl case XPT_RESET_BUS: /* Reset the specified SCSI bus */ 2611195534Sscottl case XPT_RESET_DEV: /* Bus Device Reset the specified SCSI device */ 2612279918Smav ahci_reset(ch); 2613195534Sscottl ccb->ccb_h.status = CAM_REQ_CMP; 2614195534Sscottl break; 2615195534Sscottl case XPT_TERM_IO: /* Terminate the I/O process */ 2616195534Sscottl /* XXX Implement */ 2617195534Sscottl ccb->ccb_h.status = CAM_REQ_INVALID; 2618195534Sscottl break; 2619195534Sscottl case XPT_PATH_INQ: /* Path routing inquiry */ 2620195534Sscottl { 2621195534Sscottl struct ccb_pathinq *cpi = &ccb->cpi; 2622195534Sscottl 2623195534Sscottl cpi->version_num = 1; /* XXX??? */ 2624199278Smav cpi->hba_inquiry = PI_SDTR_ABLE; 2625199278Smav if (ch->caps & AHCI_CAP_SNCQ) 2626199278Smav cpi->hba_inquiry |= PI_TAG_ABLE; 2627195534Sscottl if (ch->caps & AHCI_CAP_SPM) 2628195534Sscottl cpi->hba_inquiry |= PI_SATAPM; 2629195534Sscottl cpi->target_sprt = 0; 2630248522Skib cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; 2631195534Sscottl cpi->hba_eng_cnt = 0; 2632195534Sscottl if (ch->caps & AHCI_CAP_SPM) 2633198322Smav cpi->max_target = 15; 2634195534Sscottl else 2635195534Sscottl cpi->max_target = 0; 2636195534Sscottl cpi->max_lun = 0; 2637195534Sscottl cpi->initiator_id = 0; 2638195534Sscottl cpi->bus_id = cam_sim_bus(sim); 2639195534Sscottl cpi->base_transfer_speed = 150000; 2640315813Smav strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 2641315813Smav strlcpy(cpi->hba_vid, "AHCI", HBA_IDLEN); 2642315813Smav strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 2643195534Sscottl cpi->unit_number = cam_sim_unit(sim); 2644195534Sscottl cpi->transport = XPORT_SATA; 2645196656Smav cpi->transport_version = XPORT_VERSION_UNSPECIFIED; 2646236847Smav cpi->protocol = PROTO_ATA; 2647196656Smav cpi->protocol_version = PROTO_VERSION_UNSPECIFIED; 2648195534Sscottl cpi->maxio = MAXPHYS; 2649196777Smav /* ATI SB600 can't handle 256 sectors with FPDMA (NCQ). */ 2650279918Smav if (ch->quirks & AHCI_Q_MAXIO_64K) 2651196796Smav cpi->maxio = min(cpi->maxio, 128 * 512); 2652279918Smav cpi->hba_vendor = ch->vendorid; 2653279918Smav cpi->hba_device = ch->deviceid; 2654279918Smav cpi->hba_subvendor = ch->subvendorid; 2655279918Smav cpi->hba_subdevice = ch->subdeviceid; 2656195534Sscottl cpi->ccb_h.status = CAM_REQ_CMP; 2657195534Sscottl break; 2658195534Sscottl } 2659195534Sscottl default: 2660195534Sscottl ccb->ccb_h.status = CAM_REQ_INVALID; 2661195534Sscottl break; 2662195534Sscottl } 2663260387Sscottl ahci_done(ch, ccb); 2664195534Sscottl} 2665195534Sscottl 2666195534Sscottlstatic void 2667195534Sscottlahcipoll(struct cam_sim *sim) 2668195534Sscottl{ 2669195534Sscottl struct ahci_channel *ch = (struct ahci_channel *)cam_sim_softc(sim); 2670260387Sscottl uint32_t istatus; 2671195534Sscottl 2672260387Sscottl /* Read interrupt statuses and process if any. */ 2673260387Sscottl istatus = ATA_INL(ch->r_mem, AHCI_P_IS); 2674260387Sscottl if (istatus != 0) 2675260387Sscottl ahci_ch_intr_main(ch, istatus); 2676220789Smav if (ch->resetting != 0 && 2677220789Smav (--ch->resetpolldiv <= 0 || !callout_pending(&ch->reset_timer))) { 2678220789Smav ch->resetpolldiv = 1000; 2679279918Smav ahci_reset_to(ch); 2680220789Smav } 2681195534Sscottl} 2682279918SmavMODULE_VERSION(ahci, 1); 2683279918SmavMODULE_DEPEND(ahci, cam, 1, 1, 1); 2684