| File: | dev/ic/ahci.c |
| Warning: | line 3416, column 10 Although the value stored to 'rc' is used in the enclosing expression, the value is never actually read from 'rc' |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* $OpenBSD: ahci.c,v 1.39 2023/02/03 18:31:16 miod Exp $ */ |
| 2 | |
| 3 | /* |
| 4 | * Copyright (c) 2006 David Gwynne <dlg@openbsd.org> |
| 5 | * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com> |
| 6 | * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org> |
| 7 | * |
| 8 | * Permission to use, copy, modify, and distribute this software for any |
| 9 | * purpose with or without fee is hereby granted, provided that the above |
| 10 | * copyright notice and this permission notice appear in all copies. |
| 11 | * |
| 12 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 13 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 14 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
| 15 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 16 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 17 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
| 18 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 19 | */ |
| 20 | |
| 21 | #include <sys/param.h> |
| 22 | #include <sys/systm.h> |
| 23 | #include <sys/buf.h> |
| 24 | #include <sys/kernel.h> |
| 25 | #include <sys/malloc.h> |
| 26 | #include <sys/device.h> |
| 27 | #include <sys/queue.h> |
| 28 | #include <sys/mutex.h> |
| 29 | #include <sys/pool.h> |
| 30 | |
| 31 | #include <machine/bus.h> |
| 32 | |
| 33 | #include <dev/ic/ahcireg.h> |
| 34 | #include <dev/ic/ahcivar.h> |
| 35 | |
| 36 | #ifdef AHCI_DEBUG |
| 37 | #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \ |
| 38 | while (0) |
| 39 | #define AHCI_D_TIMEOUT 0x00 |
| 40 | #define AHCI_D_VERBOSE 0x01 |
| 41 | #define AHCI_D_INTR 0x02 |
| 42 | #define AHCI_D_XFER 0x08 |
| 43 | int ahcidebug = AHCI_D_VERBOSE; |
| 44 | #else |
| 45 | #define DPRINTF(m, f...) |
| 46 | #endif |
| 47 | |
| 48 | #ifdef HIBERNATE1 |
| 49 | #include <uvm/uvm_extern.h> |
| 50 | #include <sys/hibernate.h> |
| 51 | #include <sys/disk.h> |
| 52 | #include <sys/disklabel.h> |
| 53 | |
| 54 | #include <scsi/scsi_all.h> |
| 55 | #include <scsi/scsiconf.h> |
| 56 | |
| 57 | void ahci_hibernate_io_start(struct ahci_port *, |
| 58 | struct ahci_ccb *); |
| 59 | int ahci_hibernate_io_poll(struct ahci_port *, |
| 60 | struct ahci_ccb *); |
| 61 | void ahci_hibernate_load_prdt(struct ahci_ccb *); |
| 62 | |
| 63 | int ahci_hibernate_io(dev_t dev, daddr_t blkno, |
| 64 | vaddr_t addr, size_t size, int wr, void *page); |
| 65 | #endif |
| 66 | |
| 67 | struct cfdriver ahci_cd = { |
| 68 | NULL((void *)0), "ahci", DV_DULL |
| 69 | }; |
| 70 | |
| 71 | void ahci_enable_interrupts(struct ahci_port *); |
| 72 | |
| 73 | int ahci_init(struct ahci_softc *); |
| 74 | int ahci_port_alloc(struct ahci_softc *, u_int); |
| 75 | void ahci_port_detect(struct ahci_softc *, u_int); |
| 76 | void ahci_port_free(struct ahci_softc *, u_int); |
| 77 | int ahci_port_init(struct ahci_softc *, u_int); |
| 78 | |
| 79 | int ahci_default_port_start(struct ahci_port *, int); |
| 80 | int ahci_port_stop(struct ahci_port *, int); |
| 81 | int ahci_port_clo(struct ahci_port *); |
| 82 | int ahci_port_softreset(struct ahci_port *); |
| 83 | void ahci_port_comreset(struct ahci_port *); |
| 84 | int ahci_port_portreset(struct ahci_port *, int); |
| 85 | void ahci_port_portreset_start(struct ahci_port *); |
| 86 | int ahci_port_portreset_poll(struct ahci_port *); |
| 87 | void ahci_port_portreset_wait(struct ahci_port *); |
| 88 | int ahci_port_portreset_finish(struct ahci_port *, int); |
| 89 | int ahci_port_signature(struct ahci_port *); |
| 90 | int ahci_pmp_port_softreset(struct ahci_port *, int); |
| 91 | int ahci_pmp_port_portreset(struct ahci_port *, int); |
| 92 | int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port); |
| 93 | |
| 94 | int ahci_load_prdt(struct ahci_ccb *); |
| 95 | void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t, |
| 96 | u_int32_t, u_int32_t); |
| 97 | void ahci_unload_prdt(struct ahci_ccb *); |
| 98 | |
| 99 | int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); |
| 100 | void ahci_start(struct ahci_ccb *); |
| 101 | |
| 102 | void ahci_issue_pending_ncq_commands(struct ahci_port *); |
| 103 | void ahci_issue_pending_commands(struct ahci_port *, int); |
| 104 | |
| 105 | int ahci_intr(void *); |
| 106 | u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); |
| 107 | |
| 108 | struct ahci_ccb *ahci_get_ccb(struct ahci_port *); |
| 109 | void ahci_put_ccb(struct ahci_ccb *); |
| 110 | |
| 111 | struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); |
| 112 | void ahci_put_err_ccb(struct ahci_ccb *); |
| 113 | |
| 114 | struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *); |
| 115 | void ahci_put_pmp_ccb(struct ahci_ccb *); |
| 116 | |
| 117 | int ahci_port_read_ncq_error(struct ahci_port *, int *, int); |
| 118 | |
| 119 | struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t); |
| 120 | void ahci_dmamem_free(struct ahci_softc *, |
| 121 | struct ahci_dmamem *); |
| 122 | |
| 123 | u_int32_t ahci_read(struct ahci_softc *, bus_size_t); |
| 124 | void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); |
| 125 | int ahci_wait_ne(struct ahci_softc *, bus_size_t, |
| 126 | u_int32_t, u_int32_t); |
| 127 | |
| 128 | u_int32_t ahci_pread(struct ahci_port *, bus_size_t); |
| 129 | void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); |
| 130 | int ahci_pwait_eq(struct ahci_port *, bus_size_t, |
| 131 | u_int32_t, u_int32_t, int); |
| 132 | void ahci_flush_tfd(struct ahci_port *ap); |
| 133 | u_int32_t ahci_active_mask(struct ahci_port *); |
| 134 | int ahci_port_detect_pmp(struct ahci_port *); |
| 135 | void ahci_pmp_probe_timeout(void *); |
| 136 | |
| 137 | /* pmp operations */ |
| 138 | int ahci_pmp_read(struct ahci_port *, int, int, |
| 139 | u_int32_t *); |
| 140 | int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t); |
| 141 | int ahci_pmp_phy_status(struct ahci_port *, int, |
| 142 | u_int32_t *); |
| 143 | int ahci_pmp_identify(struct ahci_port *, int *); |
| 144 | |
| 145 | |
| 146 | /* Wait for all bits in _b to be cleared */ |
| 147 | #define ahci_pwait_clr(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) \ |
| 148 | ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) |
| 149 | |
| 150 | /* Wait for all bits in _b to be set */ |
| 151 | #define ahci_pwait_set(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) \ |
| 152 | ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) |
| 153 | |
| 154 | |
| 155 | |
| 156 | /* provide methods for atascsi to call */ |
| 157 | int ahci_ata_probe(void *, int, int); |
| 158 | void ahci_ata_free(void *, int, int); |
| 159 | struct ata_xfer * ahci_ata_get_xfer(void *, int); |
| 160 | void ahci_ata_put_xfer(struct ata_xfer *); |
| 161 | void ahci_ata_cmd(struct ata_xfer *); |
| 162 | |
| 163 | const struct atascsi_methods ahci_atascsi_methods = { |
| 164 | ahci_ata_probe, |
| 165 | ahci_ata_free, |
| 166 | ahci_ata_get_xfer, |
| 167 | ahci_ata_put_xfer, |
| 168 | ahci_ata_cmd |
| 169 | }; |
| 170 | |
| 171 | /* ccb completions */ |
| 172 | void ahci_ata_cmd_done(struct ahci_ccb *); |
| 173 | void ahci_pmp_cmd_done(struct ahci_ccb *); |
| 174 | void ahci_ata_cmd_timeout(void *); |
| 175 | void ahci_empty_done(struct ahci_ccb *); |
| 176 | |
| 177 | int |
| 178 | ahci_attach(struct ahci_softc *sc) |
| 179 | { |
| 180 | struct atascsi_attach_args aaa; |
| 181 | u_int32_t pi; |
| 182 | int i, j, done; |
| 183 | |
| 184 | if (sc->sc_port_start == NULL((void *)0)) |
| 185 | sc->sc_port_start = ahci_default_port_start; |
| 186 | |
| 187 | if (ahci_init(sc) != 0) { |
| 188 | /* error already printed by ahci_init */ |
| 189 | goto unmap; |
| 190 | } |
| 191 | |
| 192 | printf("\n"); |
| 193 | |
| 194 | sc->sc_cap = ahci_read(sc, AHCI_REG_CAP0x000); |
| 195 | sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap)((((sc->sc_cap) & 0x1f00)>>8)+1); |
| 196 | #ifdef AHCI_DEBUG |
| 197 | if (ahcidebug & AHCI_D_VERBOSE) { |
| 198 | const char *gen; |
| 199 | |
| 200 | switch (sc->sc_cap & AHCI_REG_CAP_ISS(0xf<<20)) { |
| 201 | case AHCI_REG_CAP_ISS_G1(0x1<<20): |
| 202 | gen = "1 (1.5Gbps)"; |
| 203 | break; |
| 204 | case AHCI_REG_CAP_ISS_G2(0x2<<20): |
| 205 | gen = "2 (3.0Gb/s)"; |
| 206 | break; |
| 207 | case AHCI_REG_CAP_ISS_G3(0x3<<20): |
| 208 | gen = "3 (6.0Gb/s)"; |
| 209 | break; |
| 210 | default: |
| 211 | gen = "unknown"; |
| 212 | break; |
| 213 | } |
| 214 | |
| 215 | printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n", |
| 216 | DEVNAME(sc)((sc)->sc_dev.dv_xname), sc->sc_cap, AHCI_FMT_CAP"\020" "\040S64A" "\037NCQ" "\036SSNTF" "\035SMPS" "\034SSS" "\033SALP" "\032SAL" "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" "\021FBSS" "\020PMD" "\017SSC" "\016PSC" "\010CCCS" "\007EMS" "\006SXS", |
| 217 | AHCI_REG_CAP_NP(sc->sc_cap)(((sc->sc_cap) & 0x1f)+1), sc->sc_ncmds, gen); |
| 218 | printf("%s: extended capabilities 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), |
| 219 | ahci_read(sc, AHCI_REG_CAP20x024), AHCI_FMT_CAP2"\020" "\006DESO" "\005SADM" "\004SDS" "\003APST" "\002NVMP" "\001BOH"); |
| 220 | } |
| 221 | #endif |
| 222 | |
| 223 | pi = ahci_read(sc, AHCI_REG_PI0x00c); |
| 224 | DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", |
| 225 | DEVNAME(sc), pi); |
| 226 | |
| 227 | #ifdef AHCI_COALESCE |
| 228 | /* Naive coalescing support - enable for all ports. */ |
| 229 | if (sc->sc_cap & AHCI_REG_CAP_CCCS(1<<7)) { |
| 230 | u_int16_t ccc_timeout = 20; |
| 231 | u_int8_t ccc_numcomplete = 12; |
| 232 | u_int32_t ccc_ctl; |
| 233 | |
| 234 | /* disable coalescing during reconfiguration. */ |
| 235 | ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL0x014); |
| 236 | ccc_ctl &= ~0x00000001; |
| 237 | ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl); |
| 238 | |
| 239 | sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl)(((ccc_ctl) & 0xf8) >> 3); |
| 240 | if (pi & sc->sc_ccc_mask) { |
| 241 | /* A conflict with the implemented port list? */ |
| 242 | printf("%s: coalescing interrupt/implemented port list " |
| 243 | "conflict, PI: %08x, ccc_mask: %08x\n", |
| 244 | DEVNAME(sc)((sc)->sc_dev.dv_xname), pi, sc->sc_ccc_mask); |
| 245 | sc->sc_ccc_mask = 0; |
| 246 | goto noccc; |
| 247 | } |
| 248 | |
| 249 | /* ahci_port_start will enable each port when it starts. */ |
| 250 | sc->sc_ccc_ports = pi; |
| 251 | sc->sc_ccc_ports_cur = 0; |
| 252 | |
| 253 | /* program thresholds and enable overall coalescing. */ |
| 254 | ccc_ctl &= ~0xffffff00; |
| 255 | ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); |
| 256 | ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl); |
| 257 | ahci_write(sc, AHCI_REG_CCC_PORTS0x018, 0); |
| 258 | ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl | 1); |
| 259 | } |
| 260 | noccc: |
| 261 | #endif |
| 262 | /* |
| 263 | * Given that ahci_port_alloc() will grab one CCB for error recovery |
| 264 | * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds |
| 265 | * pretend at least 2 command slots for devices without NCQ support. |
| 266 | * That way, also at least 1 slot is made available for atascsi(4). |
| 267 | */ |
| 268 | sc->sc_ncmds = max(2, sc->sc_ncmds); |
| 269 | for (i = 0; i < AHCI_MAX_PORTS32; i++) { |
| 270 | if (!ISSET(pi, 1 << i)((pi) & (1 << i))) { |
| 271 | /* dont allocate stuff if the port isnt implemented */ |
| 272 | continue; |
| 273 | } |
| 274 | |
| 275 | if (ahci_port_alloc(sc, i) == ENOMEM12) |
| 276 | goto freeports; |
| 277 | |
| 278 | if (sc->sc_ports[i] != NULL((void *)0)) |
| 279 | ahci_port_portreset_start(sc->sc_ports[i]); |
| 280 | } |
| 281 | |
| 282 | /* |
| 283 | * Poll for device detection until all ports report a device, or one |
| 284 | * second has elapsed. |
| 285 | */ |
| 286 | for (i = 0; i < 1000; i++) { |
| 287 | done = 1; |
| 288 | for (j = 0; j < AHCI_MAX_PORTS32; j++) { |
| 289 | if (sc->sc_ports[j] == NULL((void *)0)) |
| 290 | continue; |
| 291 | |
| 292 | if (ahci_port_portreset_poll(sc->sc_ports[j])) |
| 293 | done = 0; |
| 294 | } |
| 295 | |
| 296 | if (done) |
| 297 | break; |
| 298 | |
| 299 | delay(1000)(*delay_func)(1000); |
| 300 | } |
| 301 | |
| 302 | /* |
| 303 | * Finish device detection on all ports that initialized. |
| 304 | */ |
| 305 | for (i = 0; i < AHCI_MAX_PORTS32; i++) { |
| 306 | if (sc->sc_ports[i] != NULL((void *)0)) |
| 307 | ahci_port_detect(sc, i); |
| 308 | } |
| 309 | |
| 310 | memset(&aaa, 0, sizeof(aaa))__builtin_memset((&aaa), (0), (sizeof(aaa))); |
| 311 | aaa.aaa_cookie = sc; |
| 312 | aaa.aaa_methods = &ahci_atascsi_methods; |
| 313 | aaa.aaa_minphys = NULL((void *)0); |
| 314 | aaa.aaa_nports = AHCI_MAX_PORTS32; |
| 315 | aaa.aaa_ncmds = sc->sc_ncmds - 1; |
| 316 | if (!(sc->sc_flags & AHCI_F_NO_NCQ(1<<0)) && |
| 317 | sc->sc_ncmds > 2 && |
| 318 | (sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30))) { |
| 319 | aaa.aaa_capability |= ASAA_CAP_NCQ(1 << 0) | ASAA_CAP_PMP_NCQ(1 << 2); |
| 320 | } |
| 321 | |
| 322 | sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa); |
| 323 | |
| 324 | /* Enable interrupts */ |
| 325 | ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) | AHCI_REG_GHC_IE(1<<1)); |
| 326 | |
| 327 | return 0; |
| 328 | |
| 329 | freeports: |
| 330 | for (i = 0; i < AHCI_MAX_PORTS32; i++) { |
| 331 | if (sc->sc_ports[i] != NULL((void *)0)) |
| 332 | ahci_port_free(sc, i); |
| 333 | } |
| 334 | unmap: |
| 335 | /* Disable controller */ |
| 336 | ahci_write(sc, AHCI_REG_GHC0x004, 0); |
| 337 | return 1; |
| 338 | } |
| 339 | |
| 340 | int |
| 341 | ahci_detach(struct ahci_softc *sc, int flags) |
| 342 | { |
| 343 | int rv, i; |
| 344 | |
| 345 | if (sc->sc_atascsi != NULL((void *)0)) { |
| 346 | rv = atascsi_detach(sc->sc_atascsi, flags); |
| 347 | if (rv != 0) |
| 348 | return (rv); |
| 349 | } |
| 350 | |
| 351 | for (i = 0; i < AHCI_MAX_PORTS32; i++) { |
| 352 | if (sc->sc_ports[i] != NULL((void *)0)) |
| 353 | ahci_port_free(sc, i); |
| 354 | } |
| 355 | |
| 356 | return (0); |
| 357 | } |
| 358 | |
| 359 | int |
| 360 | ahci_activate(struct device *self, int act) |
| 361 | { |
| 362 | struct ahci_softc *sc = (struct ahci_softc *)self; |
| 363 | int i, rv = 0; |
| 364 | |
| 365 | switch (act) { |
| 366 | case DVACT_RESUME4: |
| 367 | /* enable ahci (global interrupts disabled) */ |
| 368 | ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31)); |
| 369 | |
| 370 | /* restore BIOS initialised parameters */ |
| 371 | ahci_write(sc, AHCI_REG_CAP0x000, sc->sc_cap); |
| 372 | |
| 373 | for (i = 0; i < AHCI_MAX_PORTS32; i++) { |
| 374 | if (sc->sc_ports[i] != NULL((void *)0)) |
| 375 | ahci_port_init(sc, i); |
| 376 | } |
| 377 | |
| 378 | /* Enable interrupts */ |
| 379 | ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) | AHCI_REG_GHC_IE(1<<1)); |
| 380 | |
| 381 | rv = config_activate_children(self, act); |
| 382 | break; |
| 383 | case DVACT_POWERDOWN6: |
| 384 | rv = config_activate_children(self, act); |
| 385 | for (i = 0; i < AHCI_MAX_PORTS32; i++) { |
| 386 | if (sc->sc_ports[i] != NULL((void *)0)) |
| 387 | ahci_port_stop(sc->sc_ports[i], 1); |
| 388 | } |
| 389 | break; |
| 390 | default: |
| 391 | rv = config_activate_children(self, act); |
| 392 | break; |
| 393 | } |
| 394 | return (rv); |
| 395 | } |
| 396 | |
| 397 | int |
| 398 | ahci_init(struct ahci_softc *sc) |
| 399 | { |
| 400 | u_int32_t reg, cap, pi; |
| 401 | const char *revision; |
| 402 | |
| 403 | DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC), |
| 404 | AHCI_FMT_GHC); |
| 405 | |
| 406 | /* save BIOS initialised parameters, enable staggered spin up */ |
| 407 | cap = ahci_read(sc, AHCI_REG_CAP0x000); |
| 408 | cap &= AHCI_REG_CAP_SMPS(1<<28); |
| 409 | cap |= AHCI_REG_CAP_SSS(1<<27); |
| 410 | pi = ahci_read(sc, AHCI_REG_PI0x00c); |
| 411 | |
| 412 | if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))(((1<<31)) & (ahci_read(sc, 0x004)))) { |
| 413 | /* reset the controller */ |
| 414 | ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0)); |
| 415 | if (ahci_wait_ne(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0), |
| 416 | AHCI_REG_GHC_HR(1<<0)) != 0) { |
| 417 | printf(" unable to reset controller\n"); |
| 418 | return (1); |
| 419 | } |
| 420 | } |
| 421 | |
| 422 | /* enable ahci (global interrupts disabled) */ |
| 423 | ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31)); |
| 424 | |
| 425 | /* restore parameters */ |
| 426 | ahci_write(sc, AHCI_REG_CAP0x000, cap); |
| 427 | ahci_write(sc, AHCI_REG_PI0x00c, pi); |
| 428 | |
| 429 | /* check the revision */ |
| 430 | reg = ahci_read(sc, AHCI_REG_VS0x010); |
| 431 | switch (reg) { |
| 432 | case AHCI_REG_VS_0_950x00000905: |
| 433 | revision = "0.95"; |
| 434 | break; |
| 435 | case AHCI_REG_VS_1_00x00010000: |
| 436 | revision = "1.0"; |
| 437 | break; |
| 438 | case AHCI_REG_VS_1_10x00010100: |
| 439 | revision = "1.1"; |
| 440 | break; |
| 441 | case AHCI_REG_VS_1_20x00010200: |
| 442 | revision = "1.2"; |
| 443 | break; |
| 444 | case AHCI_REG_VS_1_30x00010300: |
| 445 | revision = "1.3"; |
| 446 | break; |
| 447 | case AHCI_REG_VS_1_3_10x00010301: |
| 448 | revision = "1.3.1"; |
| 449 | break; |
| 450 | |
| 451 | default: |
| 452 | printf(" unsupported AHCI revision 0x%08x\n", reg); |
| 453 | return (1); |
| 454 | } |
| 455 | |
| 456 | printf(" AHCI %s", revision); |
| 457 | |
| 458 | return (0); |
| 459 | } |
| 460 | |
| 461 | void |
| 462 | ahci_enable_interrupts(struct ahci_port *ap) |
| 463 | { |
| 464 | ahci_pwrite(ap, AHCI_PREG_IE0x14, AHCI_PREG_IE_TFEE(1<<30) | AHCI_PREG_IE_HBFE(1<<29) | |
| 465 | AHCI_PREG_IE_IFE(1<<27) | AHCI_PREG_IE_OFE(1<<24) | AHCI_PREG_IE_DPE(1<<5) | |
| 466 | AHCI_PREG_IE_UFE(1<<4) | |
| 467 | ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29)) ? AHCI_PREG_IE_IPME(1<<23) : 0) | |
| 468 | #ifdef AHCI_COALESCE |
| 469 | ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 : |
| 470 | (AHCI_PREG_IE_SDBE(1<<3) | AHCI_PREG_IE_DHRE(1<<0))) |
| 471 | #else |
| 472 | AHCI_PREG_IE_SDBE(1<<3) | AHCI_PREG_IE_DHRE(1<<0) |
| 473 | #endif |
| 474 | ); |
| 475 | } |
| 476 | |
| 477 | int |
| 478 | ahci_port_alloc(struct ahci_softc *sc, u_int port) |
| 479 | { |
| 480 | struct ahci_port *ap; |
| 481 | struct ahci_ccb *ccb; |
| 482 | u_int64_t dva; |
| 483 | u_int32_t cmd; |
| 484 | struct ahci_cmd_hdr *hdr; |
| 485 | struct ahci_cmd_table *table; |
| 486 | int i, rc = ENOMEM12; |
| 487 | |
| 488 | ap = malloc(sizeof(*ap), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008); |
| 489 | if (ap == NULL((void *)0)) { |
| 490 | printf("%s: unable to allocate memory for port %d\n", |
| 491 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 492 | goto reterr; |
| 493 | } |
| 494 | ap->ap_err_scratch = dma_alloc(DEV_BSIZE(1 << 9), PR_NOWAIT0x0002 | PR_ZERO0x0008); |
| 495 | if (ap->ap_err_scratch == NULL((void *)0)) { |
| 496 | printf("%s: unable to allocate DMA scratch buf for port %d\n", |
| 497 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 498 | free(ap, M_DEVBUF2, sizeof(*ap)); |
| 499 | goto reterr; |
| 500 | } |
| 501 | |
| 502 | #ifdef AHCI_DEBUG |
| 503 | snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", |
| 504 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 505 | #endif |
| 506 | ap->ap_port = port; |
| 507 | sc->sc_ports[port] = ap; |
| 508 | |
| 509 | if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, |
| 510 | AHCI_PORT_REGION(port)(0x100 + ((port) * 0x80)), AHCI_PORT_SIZE0x80, &ap->ap_ioh) != 0) { |
| 511 | printf("%s: unable to create register window for port %d\n", |
| 512 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 513 | goto freeport; |
| 514 | } |
| 515 | |
| 516 | ap->ap_sc = sc; |
| 517 | #ifdef AHCI_COALESCE |
| 518 | ap->ap_num = port; |
| 519 | #endif |
| 520 | TAILQ_INIT(&ap->ap_ccb_free)do { (&ap->ap_ccb_free)->tqh_first = ((void *)0); ( &ap->ap_ccb_free)->tqh_last = &(&ap->ap_ccb_free )->tqh_first; } while (0); |
| 521 | TAILQ_INIT(&ap->ap_ccb_pending)do { (&ap->ap_ccb_pending)->tqh_first = ((void *)0) ; (&ap->ap_ccb_pending)->tqh_last = &(&ap-> ap_ccb_pending)->tqh_first; } while (0); |
| 522 | mtx_init(&ap->ap_ccb_mtx, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&ap-> ap_ccb_mtx), ((((0x3)) > 0x0 && ((0x3)) < 0x9) ? 0x9 : ((0x3)))); } while (0); |
| 523 | |
| 524 | /* Disable port interrupts */ |
| 525 | ahci_pwrite(ap, AHCI_PREG_IE0x14, 0); |
| 526 | |
| 527 | /* Sec 10.1.2 - deinitialise port if it is already running */ |
| 528 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18); |
| 529 | if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |((cmd) & (((1<<0) | (1<<15) | (1<<4) | ( 1<<14)))) |
| 530 | AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR))((cmd) & (((1<<0) | (1<<15) | (1<<4) | ( 1<<14)))) || |
| 531 | ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) { |
| 532 | int r; |
| 533 | |
| 534 | r = ahci_port_stop(ap, 1); |
| 535 | if (r) { |
| 536 | printf("%s: unable to disable %s, ignoring port %d\n", |
| 537 | DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port); |
| 538 | rc = ENXIO6; |
| 539 | goto freeport; |
| 540 | } |
| 541 | |
| 542 | /* Write DET to zero */ |
| 543 | ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0); |
| 544 | } |
| 545 | |
| 546 | /* Allocate RFIS */ |
| 547 | ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis)); |
| 548 | if (ap->ap_dmamem_rfis == NULL((void *)0)) |
| 549 | goto nomem; |
| 550 | |
| 551 | /* Setup RFIS base address */ |
| 552 | ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void *)(ap->ap_dmamem_rfis)->adm_kva); |
| 553 | dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0 ].ds_addr); |
| 554 | ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32)); |
| 555 | ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva); |
| 556 | |
| 557 | /* Enable FIS reception and activate port. */ |
| 558 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 559 | cmd |= AHCI_PREG_CMD_FRE(1<<4) | AHCI_PREG_CMD_POD(1<<2) | AHCI_PREG_CMD_SUD(1<<1); |
| 560 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd | AHCI_PREG_CMD_ICC_ACTIVE0x10000000); |
| 561 | |
| 562 | /* Check whether port activated. Skip it if not. */ |
| 563 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 564 | if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) { |
| 565 | rc = ENXIO6; |
| 566 | goto freeport; |
| 567 | } |
| 568 | |
| 569 | /* Allocate a CCB for each command slot */ |
| 570 | ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb), |
| 571 | M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008); |
| 572 | if (ap->ap_ccbs == NULL((void *)0)) { |
| 573 | printf("%s: unable to allocate command list for port %d\n", |
| 574 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 575 | goto freeport; |
| 576 | } |
| 577 | |
| 578 | /* Command List Structures and Command Tables */ |
| 579 | ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, |
| 580 | sc->sc_ncmds * sizeof(struct ahci_cmd_hdr)); |
| 581 | ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, |
| 582 | sc->sc_ncmds * sizeof(struct ahci_cmd_table)); |
| 583 | if (ap->ap_dmamem_cmd_table == NULL((void *)0) || ap->ap_dmamem_cmd_list == NULL((void *)0)) { |
| 584 | nomem: |
| 585 | printf("%s: unable to allocate DMA memory for port %d\n", |
| 586 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 587 | goto freeport; |
| 588 | } |
| 589 | |
| 590 | /* Setup command list base address */ |
| 591 | dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs [0].ds_addr); |
| 592 | ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32)); |
| 593 | ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva); |
| 594 | |
| 595 | /* Split CCB allocation into CCBs and assign to command header/table */ |
| 596 | hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list)((void *)(ap->ap_dmamem_cmd_list)->adm_kva); |
| 597 | table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table)((void *)(ap->ap_dmamem_cmd_table)->adm_kva); |
| 598 | for (i = 0; i < sc->sc_ncmds; i++) { |
| 599 | ccb = &ap->ap_ccbs[i]; |
| 600 | |
| 601 | if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 * 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 | 0x0002), (&ccb->ccb_dmamap)) |
| 602 | (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 * 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 | 0x0002), (&ccb->ccb_dmamap)) |
| 603 | &ccb->ccb_dmamap)(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 * 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 | 0x0002), (&ccb->ccb_dmamap)) != 0) { |
| 604 | printf("%s: unable to create dmamap for port %d " |
| 605 | "ccb %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port, i); |
| 606 | goto freeport; |
| 607 | } |
| 608 | |
| 609 | ccb->ccb_slot = i; |
| 610 | ccb->ccb_port = ap; |
| 611 | ccb->ccb_cmd_hdr = &hdr[i]; |
| 612 | ccb->ccb_cmd_table = &table[i]; |
| 613 | htolem64(&ccb->ccb_cmd_hdr->ctba,(*(__uint64_t *)(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table )))) |
| 614 | AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +(*(__uint64_t *)(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table )))) |
| 615 | ccb->ccb_slot * sizeof(struct ahci_cmd_table))(*(__uint64_t *)(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table )))); |
| 616 | |
| 617 | ccb->ccb_xa.fis = |
| 618 | (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; |
| 619 | ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; |
| 620 | ccb->ccb_xa.tag = i; |
| 621 | |
| 622 | ccb->ccb_xa.state = ATA_S_COMPLETE2; |
| 623 | ahci_put_ccb(ccb); |
| 624 | } |
| 625 | |
| 626 | /* grab a ccb for use during error recovery */ |
| 627 | ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1]; |
| 628 | TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry)do { if (((ap->ap_ccb_err)->ccb_entry.tqe_next) != ((void *)0)) (ap->ap_ccb_err)->ccb_entry.tqe_next->ccb_entry .tqe_prev = (ap->ap_ccb_err)->ccb_entry.tqe_prev; else ( &ap->ap_ccb_free)->tqh_last = (ap->ap_ccb_err)-> ccb_entry.tqe_prev; *(ap->ap_ccb_err)->ccb_entry.tqe_prev = (ap->ap_ccb_err)->ccb_entry.tqe_next; ((ap->ap_ccb_err )->ccb_entry.tqe_prev) = ((void *)-1); ((ap->ap_ccb_err )->ccb_entry.tqe_next) = ((void *)-1); } while (0); |
| 629 | ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE2; |
| 630 | |
| 631 | /* Wait for ICC change to complete */ |
| 632 | ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1)); |
| 633 | rc = 0; |
| 634 | |
| 635 | freeport: |
| 636 | if (rc != 0) |
| 637 | ahci_port_free(sc, port); |
| 638 | reterr: |
| 639 | return (rc); |
| 640 | } |
| 641 | |
| 642 | void |
| 643 | ahci_port_detect(struct ahci_softc *sc, u_int port) |
| 644 | { |
| 645 | struct ahci_port *ap; |
| 646 | const char *speed; |
| 647 | int rc; |
| 648 | |
| 649 | ap = sc->sc_ports[port]; |
| 650 | |
| 651 | rc = ahci_port_portreset_finish(ap, 1); |
| 652 | switch (rc) { |
| 653 | case ENODEV19: |
| 654 | switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) { |
| 655 | case AHCI_PREG_SSTS_DET_DEV_NE0x1: |
| 656 | printf("%s: device not communicating on port %d\n", |
| 657 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 658 | break; |
| 659 | case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4: |
| 660 | printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), |
| 661 | port); |
| 662 | break; |
| 663 | default: |
| 664 | DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " |
| 665 | "on port %d\n", DEVNAME(sc), port); |
| 666 | break; |
| 667 | } |
| 668 | goto freeport; |
| 669 | |
| 670 | case EBUSY16: |
| 671 | printf("%s: device on port %d didn't come ready, " |
| 672 | "TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port, |
| 673 | ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR"); |
| 674 | |
| 675 | /* Try a soft reset to clear busy */ |
| 676 | rc = ahci_port_softreset(ap); |
| 677 | if (rc) { |
| 678 | printf("%s: unable to communicate " |
| 679 | "with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 680 | goto freeport; |
| 681 | } |
| 682 | break; |
| 683 | |
| 684 | default: |
| 685 | break; |
| 686 | } |
| 687 | |
| 688 | DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n", |
| 689 | DEVNAME(sc), port, rc); |
| 690 | |
| 691 | /* Read current link speed */ |
| 692 | switch(ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_SPD0xf0) { |
| 693 | case AHCI_PREG_SSTS_SPD_GEN10x10: |
| 694 | speed = "1.5Gb/s"; |
| 695 | break; |
| 696 | case AHCI_PREG_SSTS_SPD_GEN20x20: |
| 697 | speed = "3.0Gb/s"; |
| 698 | break; |
| 699 | case AHCI_PREG_SSTS_SPD_GEN30x30: |
| 700 | speed = "6.0Gb/s"; |
| 701 | break; |
| 702 | default: |
| 703 | speed = NULL((void *)0); |
| 704 | break; |
| 705 | } |
| 706 | if (speed != NULL((void *)0)) |
| 707 | printf("%s: port %d: %s\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), port, speed); |
| 708 | |
| 709 | /* Enable command transfers on port */ |
| 710 | if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) { |
| 711 | printf("%s: failed to start command DMA on port %d, " |
| 712 | "disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 713 | rc = ENXIO6; /* couldn't start port */ |
| 714 | } |
| 715 | |
| 716 | /* Flush interrupts for port */ |
| 717 | ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10)); |
| 718 | ahci_write(sc, AHCI_REG_IS0x008, 1 << port); |
| 719 | |
| 720 | ahci_enable_interrupts(ap); |
| 721 | freeport: |
| 722 | if (rc != 0) |
| 723 | ahci_port_free(sc, port); |
| 724 | } |
| 725 | |
| 726 | void |
| 727 | ahci_port_free(struct ahci_softc *sc, u_int port) |
| 728 | { |
| 729 | struct ahci_port *ap = sc->sc_ports[port]; |
| 730 | struct ahci_ccb *ccb; |
| 731 | |
| 732 | /* Ensure port is disabled and its interrupts are flushed */ |
| 733 | if (ap->ap_sc) { |
| 734 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, 0); |
| 735 | ahci_pwrite(ap, AHCI_PREG_IE0x14, 0); |
| 736 | ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10)); |
| 737 | ahci_write(sc, AHCI_REG_IS0x008, 1 << port); |
| 738 | } |
| 739 | |
| 740 | if (ap->ap_ccb_err) |
| 741 | ahci_put_ccb(ap->ap_ccb_err); |
| 742 | |
| 743 | if (ap->ap_ccbs) { |
| 744 | while ((ccb = ahci_get_ccb(ap)) != NULL((void *)0)) |
| 745 | bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (ccb ->ccb_dmamap)); |
| 746 | free(ap->ap_ccbs, M_DEVBUF2, sc->sc_ncmds * sizeof(*ccb)); |
| 747 | } |
| 748 | |
| 749 | if (ap->ap_dmamem_cmd_list) |
| 750 | ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); |
| 751 | if (ap->ap_dmamem_rfis) |
| 752 | ahci_dmamem_free(sc, ap->ap_dmamem_rfis); |
| 753 | if (ap->ap_dmamem_cmd_table) |
| 754 | ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); |
| 755 | if (ap->ap_err_scratch) |
| 756 | dma_free(ap->ap_err_scratch, DEV_BSIZE(1 << 9)); |
| 757 | |
| 758 | /* bus_space(9) says we dont free the subregions handle */ |
| 759 | |
| 760 | free(ap, M_DEVBUF2, sizeof(*ap)); |
| 761 | sc->sc_ports[port] = NULL((void *)0); |
| 762 | } |
| 763 | |
| 764 | int |
| 765 | ahci_port_init(struct ahci_softc *sc, u_int port) |
| 766 | { |
| 767 | struct ahci_port *ap; |
| 768 | u_int64_t dva; |
| 769 | u_int32_t cmd; |
| 770 | int rc = ENOMEM12; |
| 771 | |
| 772 | ap = sc->sc_ports[port]; |
| 773 | #ifdef AHCI_DEBUG |
| 774 | snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d", |
| 775 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 776 | #endif |
| 777 | |
| 778 | /* Disable port interrupts */ |
| 779 | ahci_pwrite(ap, AHCI_PREG_IE0x14, 0); |
| 780 | |
| 781 | /* Sec 10.1.2 - deinitialise port if it is already running */ |
| 782 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18); |
| 783 | if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |((cmd) & (((1<<0) | (1<<15) | (1<<4) | ( 1<<14)))) |
| 784 | AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR))((cmd) & (((1<<0) | (1<<15) | (1<<4) | ( 1<<14)))) || |
| 785 | ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) { |
| 786 | int r; |
| 787 | |
| 788 | r = ahci_port_stop(ap, 1); |
| 789 | if (r) { |
| 790 | printf("%s: unable to disable %s, ignoring port %d\n", |
| 791 | DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port); |
| 792 | rc = ENXIO6; |
| 793 | goto reterr; |
| 794 | } |
| 795 | |
| 796 | /* Write DET to zero */ |
| 797 | ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0); |
| 798 | } |
| 799 | |
| 800 | /* Setup RFIS base address */ |
| 801 | ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void *)(ap->ap_dmamem_rfis)->adm_kva); |
| 802 | dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0 ].ds_addr); |
| 803 | ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32)); |
| 804 | ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva); |
| 805 | |
| 806 | /* Enable FIS reception and activate port. */ |
| 807 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 808 | cmd |= AHCI_PREG_CMD_FRE(1<<4) | AHCI_PREG_CMD_POD(1<<2) | AHCI_PREG_CMD_SUD(1<<1); |
| 809 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd | AHCI_PREG_CMD_ICC_ACTIVE0x10000000); |
| 810 | |
| 811 | /* Check whether port activated. Skip it if not. */ |
| 812 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 813 | if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) { |
| 814 | rc = ENXIO6; |
| 815 | goto reterr; |
| 816 | } |
| 817 | |
| 818 | /* Setup command list base address */ |
| 819 | dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs [0].ds_addr); |
| 820 | ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32)); |
| 821 | ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva); |
| 822 | |
| 823 | /* Wait for ICC change to complete */ |
| 824 | ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1)); |
| 825 | |
| 826 | /* Reset port */ |
| 827 | rc = ahci_port_portreset(ap, 1); |
| 828 | switch (rc) { |
| 829 | case ENODEV19: |
| 830 | switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) { |
| 831 | case AHCI_PREG_SSTS_DET_DEV_NE0x1: |
| 832 | printf("%s: device not communicating on port %d\n", |
| 833 | DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 834 | break; |
| 835 | case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4: |
| 836 | printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), |
| 837 | port); |
| 838 | break; |
| 839 | default: |
| 840 | DPRINTF(AHCI_D_VERBOSE, "%s: no device detected " |
| 841 | "on port %d\n", DEVNAME(sc), port); |
| 842 | break; |
| 843 | } |
| 844 | goto reterr; |
| 845 | |
| 846 | case EBUSY16: |
| 847 | printf("%s: device on port %d didn't come ready, " |
| 848 | "TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port, |
| 849 | ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR"); |
| 850 | |
| 851 | /* Try a soft reset to clear busy */ |
| 852 | rc = ahci_port_softreset(ap); |
| 853 | if (rc) { |
| 854 | printf("%s: unable to communicate " |
| 855 | "with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 856 | goto reterr; |
| 857 | } |
| 858 | break; |
| 859 | |
| 860 | default: |
| 861 | break; |
| 862 | } |
| 863 | DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", |
| 864 | DEVNAME(sc), port); |
| 865 | |
| 866 | if (ap->ap_pmp_ports > 0) { |
| 867 | int p; |
| 868 | |
| 869 | for (p = 0; p < ap->ap_pmp_ports; p++) { |
| 870 | int sig; |
| 871 | |
| 872 | /* might need to do a portreset first here? */ |
| 873 | |
| 874 | /* softreset the port */ |
| 875 | if (ahci_pmp_port_softreset(ap, p)) { |
| 876 | printf("%s.%d: unable to probe PMP port due to" |
| 877 | " softreset failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p); |
| 878 | continue; |
| 879 | } |
| 880 | |
| 881 | sig = ahci_port_signature(ap); |
| 882 | printf("%s.%d: port signature returned %d\n", |
| 883 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p, sig); |
| 884 | } |
| 885 | } |
| 886 | |
| 887 | /* Enable command transfers on port */ |
| 888 | if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) { |
| 889 | printf("%s: failed to start command DMA on port %d, " |
| 890 | "disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port); |
| 891 | rc = ENXIO6; /* couldn't start port */ |
| 892 | } |
| 893 | |
| 894 | /* Flush interrupts for port */ |
| 895 | ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10)); |
| 896 | ahci_write(sc, AHCI_REG_IS0x008, 1 << port); |
| 897 | |
| 898 | ahci_enable_interrupts(ap); |
| 899 | |
| 900 | reterr: |
| 901 | return (rc); |
| 902 | } |
| 903 | |
| 904 | int |
| 905 | ahci_default_port_start(struct ahci_port *ap, int fre_only) |
| 906 | { |
| 907 | u_int32_t r; |
| 908 | |
| 909 | /* Turn on FRE (and ST) */ |
| 910 | r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 911 | r |= AHCI_PREG_CMD_FRE(1<<4); |
| 912 | if (!fre_only) |
| 913 | r |= AHCI_PREG_CMD_ST(1<<0); |
| 914 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, r); |
| 915 | |
| 916 | #ifdef AHCI_COALESCE |
| 917 | /* (Re-)enable coalescing on the port. */ |
| 918 | if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { |
| 919 | ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); |
| 920 | ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018, |
| 921 | ap->ap_sc->sc_ccc_ports_cur); |
| 922 | } |
| 923 | #endif |
| 924 | |
| 925 | return (0); |
| 926 | } |
| 927 | |
| 928 | int |
| 929 | ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) |
| 930 | { |
| 931 | u_int32_t r; |
| 932 | |
| 933 | #ifdef AHCI_COALESCE |
| 934 | /* Disable coalescing on the port while it is stopped. */ |
| 935 | if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { |
| 936 | ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); |
| 937 | ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018, |
| 938 | ap->ap_sc->sc_ccc_ports_cur); |
| 939 | } |
| 940 | #endif |
| 941 | |
| 942 | /* Turn off ST (and FRE) */ |
| 943 | r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 944 | r &= ~AHCI_PREG_CMD_ST(1<<0); |
| 945 | if (stop_fis_rx) |
| 946 | r &= ~AHCI_PREG_CMD_FRE(1<<4); |
| 947 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, r); |
| 948 | |
| 949 | /* Wait for CR to go off */ |
| 950 | if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)ahci_pwait_eq((ap), (0x18), ((1<<15)), 0, (1))) |
| 951 | return (1); |
| 952 | |
| 953 | /* Wait for FR to go off */ |
| 954 | if (stop_fis_rx && |
| 955 | ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)ahci_pwait_eq((ap), (0x18), ((1<<14)), 0, (1))) |
| 956 | return (2); |
| 957 | |
| 958 | return (0); |
| 959 | } |
| 960 | |
| 961 | /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ |
| 962 | int |
| 963 | ahci_port_clo(struct ahci_port *ap) |
| 964 | { |
| 965 | struct ahci_softc *sc = ap->ap_sc; |
| 966 | u_int32_t cmd; |
| 967 | |
| 968 | /* Only attempt CLO if supported by controller */ |
| 969 | if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(sc, 0x000)) & ((1<<24)))) |
| 970 | return (1); |
| 971 | |
| 972 | /* Issue CLO */ |
| 973 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 974 | #ifdef DIAGNOSTIC1 |
| 975 | if (ISSET(cmd, AHCI_PREG_CMD_ST)((cmd) & ((1<<0)))) |
| 976 | printf("%s: CLO requested while port running\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 977 | #endif |
| 978 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd | AHCI_PREG_CMD_CLO(1<<3)); |
| 979 | |
| 980 | /* Wait for completion */ |
| 981 | if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)ahci_pwait_eq((ap), (0x18), ((1<<3)), 0, (1))) { |
| 982 | printf("%s: CLO did not complete\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 983 | return (1); |
| 984 | } |
| 985 | |
| 986 | return (0); |
| 987 | } |
| 988 | |
| 989 | /* AHCI soft reset, Section 10.4.1 */ |
| 990 | int |
| 991 | ahci_port_softreset(struct ahci_port *ap) |
| 992 | { |
| 993 | struct ahci_ccb *ccb = NULL((void *)0); |
| 994 | struct ahci_cmd_hdr *cmd_slot; |
| 995 | u_int8_t *fis; |
| 996 | int s, rc = EIO5, oldstate; |
| 997 | u_int32_t cmd; |
| 998 | |
| 999 | DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); |
| 1000 | |
| 1001 | s = splbio()splraise(0x3); |
| 1002 | oldstate = ap->ap_state; |
| 1003 | ap->ap_state = AP_S_ERROR_RECOVERY3; |
| 1004 | |
| 1005 | /* Save previous command register state */ |
| 1006 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 1007 | |
| 1008 | /* Idle port */ |
| 1009 | if (ahci_port_stop(ap, 0)) { |
| 1010 | printf("%s: failed to stop port, cannot softreset\n", |
| 1011 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 1012 | goto err; |
| 1013 | } |
| 1014 | |
| 1015 | /* Request CLO if device appears hung */ |
| 1016 | if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |((ahci_pread(ap, 0x20)) & ((1<<7) | (1<<3))) |
| 1017 | AHCI_PREG_TFD_STS_DRQ)((ahci_pread(ap, 0x20)) & ((1<<7) | (1<<3)))) |
| 1018 | ahci_port_clo(ap); |
| 1019 | |
| 1020 | /* Clear port errors to permit TFD transfer */ |
| 1021 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30)); |
| 1022 | |
| 1023 | /* Restart port */ |
| 1024 | if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) { |
| 1025 | printf("%s: failed to start port, cannot softreset\n", |
| 1026 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 1027 | goto err; |
| 1028 | } |
| 1029 | |
| 1030 | /* Check whether CLO worked */ |
| 1031 | if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0, (1)) |
| 1032 | AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0, (1))) { |
| 1033 | printf("%s: CLO %s, need port reset\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 1034 | ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(ap->ap_sc, 0x000)) & ((1<<24))) |
| 1035 | ? "failed" : "unsupported"); |
| 1036 | rc = EBUSY16; |
| 1037 | goto err; |
| 1038 | } |
| 1039 | |
| 1040 | /* Prep first D2H command with SRST feature & clear busy/reset flags */ |
| 1041 | ccb = ahci_get_err_ccb(ap); |
| 1042 | cmd_slot = ccb->ccb_cmd_hdr; |
| 1043 | memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 1044 | |
| 1045 | fis = ccb->ccb_cmd_table->cfis; |
| 1046 | fis[0] = ATA_FIS_TYPE_H2D0x27; |
| 1047 | fis[15] = ATA_FIS_CONTROL_SRST0x04; |
| 1048 | |
| 1049 | cmd_slot->prdtl = 0; |
| 1050 | htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | (1<<6)))) |
| 1051 | AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | (1<<6)))) |
| 1052 | AHCI_CMD_LIST_FLAG_W)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | (1<<6)))); |
| 1053 | |
| 1054 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 1055 | if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0) |
| 1056 | goto err; |
| 1057 | |
| 1058 | /* Prep second D2H command to read status and complete reset sequence */ |
| 1059 | fis[0] = ATA_FIS_TYPE_H2D0x27; |
| 1060 | fis[15] = 0; |
| 1061 | |
| 1062 | cmd_slot->prdtl = 0; |
| 1063 | htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<6)))); |
| 1064 | |
| 1065 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 1066 | if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0) |
| 1067 | goto err; |
| 1068 | |
| 1069 | if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1 <<0)), 0, (1)) |
| 1070 | AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1 <<0)), 0, (1))) { |
| 1071 | printf("%s: device didn't come ready after reset, TFD: 0x%b\n", |
| 1072 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ahci_pread(ap, AHCI_PREG_TFD0x20), |
| 1073 | AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR"); |
| 1074 | rc = EBUSY16; |
| 1075 | goto err; |
| 1076 | } |
| 1077 | |
| 1078 | rc = 0; |
| 1079 | err: |
| 1080 | if (ccb != NULL((void *)0)) { |
| 1081 | /* Abort our command, if it failed, by stopping command DMA. */ |
| 1082 | if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) { |
| 1083 | printf("%s: stopping the port, softreset slot %d was " |
| 1084 | "still active.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot); |
| 1085 | ahci_port_stop(ap, 0); |
| 1086 | } |
| 1087 | ccb->ccb_xa.state = ATA_S_ERROR3; |
| 1088 | ahci_put_err_ccb(ccb); |
| 1089 | } |
| 1090 | |
| 1091 | /* Restore saved CMD register state */ |
| 1092 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd); |
| 1093 | ap->ap_state = oldstate; |
| 1094 | |
| 1095 | splx(s)spllower(s); |
| 1096 | |
| 1097 | return (rc); |
| 1098 | } |
| 1099 | |
| 1100 | int |
| 1101 | ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port) |
| 1102 | { |
| 1103 | struct ahci_ccb *ccb = NULL((void *)0); |
| 1104 | u_int32_t data; |
| 1105 | int count; |
| 1106 | int rc; |
| 1107 | int s; |
| 1108 | struct ahci_cmd_hdr *cmd_slot; |
| 1109 | u_int8_t *fis; |
| 1110 | |
| 1111 | s = splbio()splraise(0x3); |
| 1112 | /* ignore spurious IFS errors while resetting */ |
| 1113 | DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap)); |
| 1114 | ap->ap_pmp_ignore_ifs = 1; |
| 1115 | |
| 1116 | count = 2; |
| 1117 | rc = 0; |
| 1118 | do { |
| 1119 | if (ccb != NULL((void *)0)) { |
| 1120 | ahci_put_pmp_ccb(ccb); |
| 1121 | ccb = NULL((void *)0); |
| 1122 | } |
| 1123 | |
| 1124 | if (ahci_pmp_phy_status(ap, pmp_port, &data)) { |
| 1125 | printf("%s.%d: unable to clear PHY status\n", |
| 1126 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port); |
| 1127 | } |
| 1128 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1); |
| 1129 | /* maybe don't do this on the first loop: */ |
| 1130 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27)); |
| 1131 | ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1); |
| 1132 | |
| 1133 | /* send first softreset FIS */ |
| 1134 | ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ |
| 1135 | cmd_slot = ccb->ccb_cmd_hdr; |
| 1136 | memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 1137 | |
| 1138 | fis = ccb->ccb_cmd_table->cfis; |
| 1139 | fis[0] = ATA_FIS_TYPE_H2D0x27; |
| 1140 | fis[1] = pmp_port; |
| 1141 | fis[15] = ATA_FIS_CONTROL_SRST0x04 | ATA_FIS_CONTROL_4BIT0x08; |
| 1142 | |
| 1143 | cmd_slot->prdtl = 0; |
| 1144 | htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | (pmp_port << 12)))) |
| 1145 | AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | (pmp_port << 12)))) |
| 1146 | (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | (pmp_port << 12)))); |
| 1147 | |
| 1148 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 1149 | |
| 1150 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n", |
| 1151 | PORTNAME(ap), pmp_port); |
| 1152 | if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { |
| 1153 | printf("%s.%d: PMP port softreset cmd failed\n", |
| 1154 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port); |
| 1155 | rc = EBUSY16; |
| 1156 | if (count > 0) { |
| 1157 | /* probably delay a while to allow |
| 1158 | * it to settle down? |
| 1159 | */ |
| 1160 | } |
| 1161 | continue; |
| 1162 | } |
| 1163 | |
| 1164 | /* send signature FIS */ |
| 1165 | memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 1166 | fis[0] = ATA_FIS_TYPE_H2D0x27; |
| 1167 | fis[1] = pmp_port; |
| 1168 | fis[15] = ATA_FIS_CONTROL_4BIT0x08; |
| 1169 | |
| 1170 | cmd_slot->prdtl = 0; |
| 1171 | htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (pmp_port << 12)))) |
| 1172 | (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (pmp_port << 12)))); |
| 1173 | |
| 1174 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n", |
| 1175 | PORTNAME(ap), pmp_port); |
| 1176 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 1177 | if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { |
| 1178 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd " |
| 1179 | "failed\n", PORTNAME(ap), pmp_port); |
| 1180 | rc = EBUSY16; |
| 1181 | if (count > 0) { |
| 1182 | /* sleep a while? */ |
| 1183 | } |
| 1184 | continue; |
| 1185 | } |
| 1186 | |
| 1187 | fis[15] = 0; |
| 1188 | break; |
| 1189 | } while (count--); |
| 1190 | |
| 1191 | if (ccb != NULL((void *)0)) { |
| 1192 | ahci_put_pmp_ccb(ccb); |
| 1193 | ccb = NULL((void *)0); |
| 1194 | } |
| 1195 | |
| 1196 | /* clean up a bit */ |
| 1197 | ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1); |
| 1198 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1); |
| 1199 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27)); |
| 1200 | ap->ap_pmp_ignore_ifs = 0; |
| 1201 | DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap)); |
| 1202 | splx(s)spllower(s); |
| 1203 | |
| 1204 | return (rc); |
| 1205 | } |
| 1206 | |
| 1207 | int |
| 1208 | ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port) |
| 1209 | { |
| 1210 | int sig; |
| 1211 | |
| 1212 | ap->ap_state = AP_S_PMP_PORT_PROBE2; |
| 1213 | |
| 1214 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap), |
| 1215 | pmp_port); |
| 1216 | if (ahci_pmp_port_portreset(ap, pmp_port)) { |
| 1217 | printf("%s.%d: unable to probe PMP port; portreset failed\n", |
| 1218 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port); |
| 1219 | ap->ap_state = AP_S_NORMAL0; |
| 1220 | return (ATA_PORT_T_NONE0); |
| 1221 | } |
| 1222 | |
| 1223 | if (ahci_pmp_port_softreset(ap, pmp_port)) { |
| 1224 | printf("%s.%d: unable to probe PMP port due to softreset " |
| 1225 | "failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port); |
| 1226 | ap->ap_state = AP_S_NORMAL0; |
| 1227 | return (ATA_PORT_T_NONE0); |
| 1228 | } |
| 1229 | |
| 1230 | sig = ahci_port_signature(ap); |
| 1231 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n", |
| 1232 | PORTNAME(ap), pmp_port, sig); |
| 1233 | ap->ap_state = AP_S_NORMAL0; |
| 1234 | return (sig); |
| 1235 | } |
| 1236 | |
| 1237 | |
| 1238 | void |
| 1239 | ahci_flush_tfd(struct ahci_port *ap) |
| 1240 | { |
| 1241 | u_int32_t r; |
| 1242 | |
| 1243 | r = ahci_pread(ap, AHCI_PREG_SERR0x30); |
| 1244 | if (r & AHCI_PREG_SERR_DIAG_X(1<<10)) |
| 1245 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, AHCI_PREG_SERR_DIAG_X(1<<10)); |
| 1246 | } |
| 1247 | |
| 1248 | u_int32_t |
| 1249 | ahci_active_mask(struct ahci_port *ap) |
| 1250 | { |
| 1251 | u_int32_t mask; |
| 1252 | |
| 1253 | mask = ahci_pread(ap, AHCI_PREG_CI0x38); |
| 1254 | if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30)) |
| 1255 | mask |= ahci_pread(ap, AHCI_PREG_SACT0x34); |
| 1256 | return mask; |
| 1257 | } |
| 1258 | |
| 1259 | void |
| 1260 | ahci_pmp_probe_timeout(void *cookie) |
| 1261 | { |
| 1262 | struct ahci_ccb *ccb = cookie; |
| 1263 | struct ahci_port *ap = ccb->ccb_port; |
| 1264 | u_int32_t mask; |
| 1265 | |
| 1266 | DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap)); |
| 1267 | switch (ccb->ccb_xa.state) { |
| 1268 | case ATA_S_PENDING1: |
| 1269 | TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = ( ccb)->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = ( ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1); } while (0); |
| 1270 | ccb->ccb_xa.state = ATA_S_TIMEOUT4; |
| 1271 | break; |
| 1272 | |
| 1273 | case ATA_S_ONCHIP5: |
| 1274 | case ATA_S_ERROR3: /* currently mostly here for the ATI SBx00 quirk */ |
| 1275 | /* clear the command on-chip */ |
| 1276 | KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&((ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0" )) |
| 1277 | ap->ap_sactive == 0)((ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0" )); |
| 1278 | ahci_port_stop(ap, 0); |
| 1279 | ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0))); |
| 1280 | |
| 1281 | if (ahci_active_mask(ap) != 0) { |
| 1282 | ahci_port_stop(ap, 0); |
| 1283 | ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0))); |
| 1284 | mask = ahci_active_mask(ap); |
| 1285 | if (mask != 0) { |
| 1286 | printf("%s: ahci_pmp_probe_timeout: failed to " |
| 1287 | "clear active cmds: %08x\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 1288 | mask); |
| 1289 | } |
| 1290 | } |
| 1291 | |
| 1292 | ccb->ccb_xa.state = ATA_S_TIMEOUT4; |
| 1293 | ap->ap_active &= ~(1 << ccb->ccb_slot); |
| 1294 | KASSERT(ap->ap_active_cnt > 0)((ap->ap_active_cnt > 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1294, "ap->ap_active_cnt > 0" )); |
| 1295 | --ap->ap_active_cnt; |
| 1296 | DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, " |
| 1297 | "active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot, |
| 1298 | ap->ap_active, ap->ap_active_cnt); |
| 1299 | break; |
| 1300 | |
| 1301 | default: |
| 1302 | panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d", |
| 1303 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state); |
| 1304 | } |
| 1305 | } |
| 1306 | |
| 1307 | int |
| 1308 | ahci_port_signature(struct ahci_port *ap) |
| 1309 | { |
| 1310 | u_int32_t sig; |
| 1311 | |
| 1312 | sig = ahci_pread(ap, AHCI_PREG_SIG0x24); |
| 1313 | if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI0xeb140101 & 0xffff0000)) |
| 1314 | return (ATA_PORT_T_ATAPI2); |
| 1315 | else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER0x96690101 & |
| 1316 | 0xffff0000)) |
| 1317 | return (ATA_PORT_T_PM3); |
| 1318 | else |
| 1319 | return (ATA_PORT_T_DISK1); |
| 1320 | } |
| 1321 | |
| 1322 | int |
| 1323 | ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port) |
| 1324 | { |
| 1325 | u_int32_t cmd, data; |
| 1326 | int loop; |
| 1327 | int rc = 1; |
| 1328 | int s; |
| 1329 | |
| 1330 | s = splbio()splraise(0x3); |
| 1331 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap), |
| 1332 | pmp_port); |
| 1333 | |
| 1334 | /* Save previous command register state */ |
| 1335 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 1336 | |
| 1337 | /* turn off power management and disable the PHY */ |
| 1338 | data = AHCI_PREG_SCTL_IPM_DISABLED0x300; |
| 1339 | /* maybe add AHCI_PREG_SCTL_DET_DISABLE */ |
| 1340 | if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1)) |
| 1341 | goto err; |
| 1342 | if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data)) |
| 1343 | goto err; |
| 1344 | delay(10000)(*delay_func)(10000); |
| 1345 | |
| 1346 | /* start COMRESET */ |
| 1347 | data = AHCI_PREG_SCTL_IPM_DISABLED0x300 | AHCI_PREG_SCTL_DET_INIT0x1; |
| 1348 | if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { |
| 1349 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap), |
| 1350 | pmp_port); |
| 1351 | data |= AHCI_PREG_SCTL_SPD_GEN10x10; |
| 1352 | } else |
| 1353 | data |= AHCI_PREG_SCTL_SPD_ANY0x00; |
| 1354 | |
| 1355 | if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data)) |
| 1356 | goto err; |
| 1357 | |
| 1358 | /* give it a while to settle down */ |
| 1359 | delay(100000)(*delay_func)(100000); |
| 1360 | |
| 1361 | if (ahci_pmp_phy_status(ap, pmp_port, &data)) { |
| 1362 | printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 1363 | pmp_port); |
| 1364 | } |
| 1365 | |
| 1366 | /* start trying to negotiate */ |
| 1367 | ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1); |
| 1368 | data = AHCI_PREG_SCTL_IPM_DISABLED0x300 | AHCI_PREG_SCTL_DET_NONE0x0; |
| 1369 | if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data)) |
| 1370 | goto err; |
| 1371 | |
| 1372 | /* give it a while to detect */ |
| 1373 | for (loop = 3; loop; --loop) { |
| 1374 | if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data)) |
| 1375 | goto err; |
| 1376 | if (data & AHCI_PREG_SSTS_DET0xf) |
| 1377 | break; |
| 1378 | delay(100000)(*delay_func)(100000); |
| 1379 | } |
| 1380 | if (loop == 0) { |
| 1381 | printf("%s.%d: port is unplugged\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port); |
| 1382 | goto err; |
| 1383 | } |
| 1384 | |
| 1385 | /* give it even longer to fully negotiate */ |
| 1386 | for (loop = 30; loop; --loop) { |
| 1387 | if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data)) |
| 1388 | goto err; |
| 1389 | if ((data & AHCI_PREG_SSTS_DET0xf) == AHCI_PREG_SSTS_DET_DEV0x3) |
| 1390 | break; |
| 1391 | delay(100000)(*delay_func)(100000); |
| 1392 | } |
| 1393 | |
| 1394 | if (loop == 0) { |
| 1395 | printf("%s.%d: device is not negotiating\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 1396 | pmp_port); |
| 1397 | goto err; |
| 1398 | } |
| 1399 | |
| 1400 | /* device detected */ |
| 1401 | DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap), |
| 1402 | pmp_port); |
| 1403 | |
| 1404 | /* clean up a bit */ |
| 1405 | delay(100000)(*delay_func)(100000); |
| 1406 | ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1); |
| 1407 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1); |
| 1408 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27)); |
| 1409 | |
| 1410 | rc = 0; |
| 1411 | err: |
| 1412 | /* Restore preserved port state */ |
| 1413 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd); |
| 1414 | splx(s)spllower(s); |
| 1415 | return (rc); |
| 1416 | } |
| 1417 | |
| 1418 | /* AHCI port reset, Section 10.4.2 */ |
| 1419 | |
| 1420 | void |
| 1421 | ahci_port_comreset(struct ahci_port *ap) |
| 1422 | { |
| 1423 | u_int32_t r; |
| 1424 | |
| 1425 | r = AHCI_PREG_SCTL_IPM_DISABLED0x300 | AHCI_PREG_SCTL_DET_INIT0x1; |
| 1426 | if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) { |
| 1427 | DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap)); |
| 1428 | r |= AHCI_PREG_SCTL_SPD_GEN10x10; |
| 1429 | } else |
| 1430 | r |= AHCI_PREG_SCTL_SPD_ANY0x00; |
| 1431 | ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r); |
| 1432 | delay(10000)(*delay_func)(10000); /* wait at least 1ms for COMRESET to be sent */ |
| 1433 | r &= ~AHCI_PREG_SCTL_DET_INIT0x1; |
| 1434 | r |= AHCI_PREG_SCTL_DET_NONE0x0; |
| 1435 | ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r); |
| 1436 | delay(10000)(*delay_func)(10000); |
| 1437 | } |
| 1438 | |
| 1439 | void |
| 1440 | ahci_port_portreset_start(struct ahci_port *ap) |
| 1441 | { |
| 1442 | int s; |
| 1443 | |
| 1444 | s = splbio()splraise(0x3); |
| 1445 | DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); |
| 1446 | |
| 1447 | /* Save previous command register state */ |
| 1448 | ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 1449 | |
| 1450 | /* Clear ST, ignoring failure */ |
| 1451 | ahci_port_stop(ap, 0); |
| 1452 | |
| 1453 | /* Perform device detection */ |
| 1454 | ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0); |
| 1455 | delay(10000)(*delay_func)(10000); |
| 1456 | ahci_port_comreset(ap); |
| 1457 | splx(s)spllower(s); |
| 1458 | } |
| 1459 | |
| 1460 | int |
| 1461 | ahci_port_portreset_poll(struct ahci_port *ap) |
| 1462 | { |
| 1463 | if ((ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) != |
| 1464 | AHCI_PREG_SSTS_DET_DEV0x3) |
| 1465 | return (EAGAIN35); |
| 1466 | return (0); |
| 1467 | } |
| 1468 | |
| 1469 | void |
| 1470 | ahci_port_portreset_wait(struct ahci_port *ap) |
| 1471 | { |
| 1472 | int i; |
| 1473 | |
| 1474 | for (i = 0; i < 1000; i++) { |
| 1475 | if (ahci_port_portreset_poll(ap) == 0) |
| 1476 | break; |
| 1477 | delay(1000)(*delay_func)(1000); |
| 1478 | } |
| 1479 | } |
| 1480 | |
| 1481 | int |
| 1482 | ahci_port_portreset_finish(struct ahci_port *ap, int pmp) |
| 1483 | { |
| 1484 | int rc, s, retries = 0; |
| 1485 | |
| 1486 | s = splbio()splraise(0x3); |
| 1487 | retry: |
| 1488 | if (ahci_port_portreset_poll(ap)) { |
| 1489 | rc = ENODEV19; |
| 1490 | if (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) { |
| 1491 | /* this may be a port multiplier with no device |
| 1492 | * on port 0, so still do the pmp check if requested. |
| 1493 | */ |
| 1494 | } else { |
| 1495 | goto err; |
| 1496 | } |
| 1497 | } else { |
| 1498 | /* Clear SERR (incl X bit), so TFD can update */ |
| 1499 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30)); |
| 1500 | |
| 1501 | /* Wait for device to become ready */ |
| 1502 | if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1 <<0)), 0, (3)) |
| 1503 | AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1 <<0)), 0, (3))) { |
| 1504 | /* even if the device doesn't wake up, check if there's |
| 1505 | * a port multiplier there |
| 1506 | */ |
| 1507 | if (retries == 0) { |
| 1508 | retries = 1; |
| 1509 | ahci_port_comreset(ap); |
| 1510 | ahci_port_portreset_wait(ap); |
| 1511 | goto retry; |
| 1512 | } |
| 1513 | rc = EBUSY16; |
| 1514 | } else { |
| 1515 | rc = 0; |
| 1516 | } |
| 1517 | } |
| 1518 | |
| 1519 | if (pmp != 0) { |
| 1520 | if (ahci_port_detect_pmp(ap)) { |
| 1521 | /* reset again without pmp support */ |
| 1522 | pmp = 0; |
| 1523 | retries = 0; |
| 1524 | ahci_port_comreset(ap); |
| 1525 | ahci_port_portreset_wait(ap); |
| 1526 | goto retry; |
| 1527 | } |
| 1528 | } |
| 1529 | |
| 1530 | err: |
| 1531 | /* Restore preserved port state */ |
| 1532 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, ap->ap_saved_cmd); |
| 1533 | ap->ap_saved_cmd = 0; |
| 1534 | splx(s)spllower(s); |
| 1535 | |
| 1536 | return (rc); |
| 1537 | } |
| 1538 | |
| 1539 | int |
| 1540 | ahci_port_portreset(struct ahci_port *ap, int pmp) |
| 1541 | { |
| 1542 | ahci_port_portreset_start(ap); |
| 1543 | ahci_port_portreset_wait(ap); |
| 1544 | return (ahci_port_portreset_finish(ap, pmp)); |
| 1545 | } |
| 1546 | |
| 1547 | int |
| 1548 | ahci_port_detect_pmp(struct ahci_port *ap) |
| 1549 | { |
| 1550 | int count, pmp_rc, rc; |
| 1551 | u_int32_t r, cmd; |
| 1552 | struct ahci_cmd_hdr *cmd_slot; |
| 1553 | struct ahci_ccb *ccb = NULL((void *)0); |
| 1554 | u_int8_t *fis = NULL((void *)0); |
| 1555 | |
| 1556 | if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP(1<<2)) || |
| 1557 | !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)((ahci_read(ap->ap_sc, 0x000)) & ((1<<17)))) { |
| 1558 | return 0; |
| 1559 | } |
| 1560 | |
| 1561 | rc = 0; |
| 1562 | pmp_rc = 0; |
| 1563 | count = 2; |
| 1564 | do { |
| 1565 | DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap), |
| 1566 | count); |
| 1567 | if (ccb != NULL((void *)0)) { |
| 1568 | ahci_put_pmp_ccb(ccb); |
| 1569 | ccb = NULL((void *)0); |
| 1570 | } |
| 1571 | ahci_port_stop(ap, 0); |
| 1572 | ap->ap_state = AP_S_PMP_PROBE1; |
| 1573 | |
| 1574 | /* set PMA in cmd reg */ |
| 1575 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 1576 | if ((cmd & AHCI_PREG_CMD_PMA(1<<17)) == 0) { |
| 1577 | cmd |= AHCI_PREG_CMD_PMA(1<<17); |
| 1578 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd); |
| 1579 | } |
| 1580 | |
| 1581 | /* Flush errors and request CLO unconditionally, |
| 1582 | * then start the port |
| 1583 | */ |
| 1584 | r = ahci_pread(ap, AHCI_PREG_SERR0x30); |
| 1585 | if (r & AHCI_PREG_SERR_DIAG_X(1<<10)) |
| 1586 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, |
| 1587 | AHCI_PREG_SERR_DIAG_X(1<<10)); |
| 1588 | |
| 1589 | /* Request CLO */ |
| 1590 | ahci_port_clo(ap); |
| 1591 | |
| 1592 | /* Clear port errors to permit TFD transfer */ |
| 1593 | r = ahci_pread(ap, AHCI_PREG_SERR0x30); |
| 1594 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, r); |
| 1595 | |
| 1596 | /* Restart port */ |
| 1597 | if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) { |
| 1598 | rc = EBUSY16; |
| 1599 | printf("%s: failed to start port, cannot probe PMP\n", |
| 1600 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 1601 | break; |
| 1602 | } |
| 1603 | |
| 1604 | /* Check whether CLO worked */ |
| 1605 | if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0, (1)) |
| 1606 | AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0, (1))) { |
| 1607 | u_int32_t cap; |
| 1608 | |
| 1609 | cap = ahci_read(ap->ap_sc, AHCI_REG_CAP0x000); |
| 1610 | printf("%s: CLO %s, need port reset\n", |
| 1611 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 1612 | ISSET(cap, AHCI_REG_CAP_SCLO)((cap) & ((1<<24))) |
| 1613 | ? "failed" : "unsupported"); |
| 1614 | pmp_rc = EBUSY16; |
| 1615 | break; |
| 1616 | } |
| 1617 | |
| 1618 | /* Prep first command with SRST feature & |
| 1619 | * clear busy/reset flags |
| 1620 | */ |
| 1621 | ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ |
| 1622 | cmd_slot = ccb->ccb_cmd_hdr; |
| 1623 | memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))) |
| 1624 | sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 1625 | |
| 1626 | fis = ccb->ccb_cmd_table->cfis; |
| 1627 | fis[0] = ATA_FIS_TYPE_H2D0x27; |
| 1628 | fis[1] = SATA_PMP_CONTROL_PORT0x0f; |
| 1629 | fis[15] = ATA_FIS_CONTROL_SRST0x04 | ATA_FIS_CONTROL_4BIT0x08; |
| 1630 | |
| 1631 | cmd_slot->prdtl = 0; |
| 1632 | htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | 0xf000))) |
| 1633 | AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | 0xf000))) |
| 1634 | AHCI_CMD_LIST_FLAG_PMP)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (1<<10) | (1<<8) | 0xf000))); |
| 1635 | |
| 1636 | DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n", |
| 1637 | PORTNAME(ap)); |
| 1638 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 1639 | if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { |
| 1640 | DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n", |
| 1641 | PORTNAME(ap)); |
| 1642 | pmp_rc = EBUSY16; |
| 1643 | continue; |
| 1644 | } |
| 1645 | |
| 1646 | if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0, (1)) |
| 1647 | AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0, (1))) { |
| 1648 | printf("%s: port busy after first PMP probe FIS\n", |
| 1649 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 1650 | } |
| 1651 | |
| 1652 | /* clear errors in case the device |
| 1653 | * didn't reset cleanly |
| 1654 | */ |
| 1655 | ahci_flush_tfd(ap); |
| 1656 | r = ahci_pread(ap, AHCI_PREG_SERR0x30); |
| 1657 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, r); |
| 1658 | |
| 1659 | /* Prep second command to read status and |
| 1660 | * complete reset sequence |
| 1661 | */ |
| 1662 | memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))) |
| 1663 | sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 1664 | fis[0] = ATA_FIS_TYPE_H2D0x27; |
| 1665 | fis[1] = SATA_PMP_CONTROL_PORT0x0f; |
| 1666 | fis[15] = ATA_FIS_CONTROL_4BIT0x08; |
| 1667 | |
| 1668 | cmd_slot->prdtl = 0; |
| 1669 | htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | 0xf000))) |
| 1670 | AHCI_CMD_LIST_FLAG_PMP)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | 0xf000))); |
| 1671 | |
| 1672 | DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n", |
| 1673 | PORTNAME(ap)); |
| 1674 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 1675 | if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) { |
| 1676 | DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status " |
| 1677 | "cmd failed\n", PORTNAME(ap)); |
| 1678 | pmp_rc = EBUSY16; |
| 1679 | continue; |
| 1680 | } |
| 1681 | |
| 1682 | /* apparently we need to retry at least once |
| 1683 | * to get the right signature |
| 1684 | */ |
| 1685 | fis[15] = 0; |
| 1686 | pmp_rc = 0; |
| 1687 | } while (--count); |
| 1688 | |
| 1689 | if (ccb != NULL((void *)0)) { |
| 1690 | ahci_put_pmp_ccb(ccb); |
| 1691 | ccb = NULL((void *)0); |
| 1692 | } |
| 1693 | |
| 1694 | if (ap->ap_state == AP_S_PMP_PROBE1) { |
| 1695 | ap->ap_state = AP_S_NORMAL0; |
| 1696 | } |
| 1697 | |
| 1698 | if (pmp_rc == 0) { |
| 1699 | if (ahci_port_signature(ap) != ATA_PORT_T_PM3) { |
| 1700 | DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n", |
| 1701 | PORTNAME(ap)); |
| 1702 | pmp_rc = EBUSY16; |
| 1703 | } else { |
| 1704 | DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n", |
| 1705 | PORTNAME(ap)); |
| 1706 | } |
| 1707 | } |
| 1708 | |
| 1709 | if (pmp_rc == 0) { |
| 1710 | if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) { |
| 1711 | pmp_rc = EBUSY16; |
| 1712 | } else { |
| 1713 | rc = 0; |
| 1714 | } |
| 1715 | } |
| 1716 | |
| 1717 | /* if PMP detection failed, so turn off the PMA bit and |
| 1718 | * reset the port again |
| 1719 | */ |
| 1720 | if (pmp_rc != 0) { |
| 1721 | DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting " |
| 1722 | "the port\n", PORTNAME(ap)); |
| 1723 | ahci_port_stop(ap, 0); |
| 1724 | ahci_port_clo(ap); |
| 1725 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 1726 | cmd &= ~AHCI_PREG_CMD_PMA(1<<17); |
| 1727 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd); |
| 1728 | |
| 1729 | ahci_pwrite(ap, AHCI_PREG_IE0x14, 0); |
| 1730 | ahci_port_stop(ap, 0); |
| 1731 | if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29)) |
| 1732 | ahci_pwrite(ap, AHCI_PREG_SNTF0x3c, -1); |
| 1733 | ahci_flush_tfd(ap); |
| 1734 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1); |
| 1735 | |
| 1736 | ahci_pwrite(ap, AHCI_PREG_IS0x10, -1); |
| 1737 | |
| 1738 | ahci_enable_interrupts(ap); |
| 1739 | |
| 1740 | rc = pmp_rc; |
| 1741 | } |
| 1742 | |
| 1743 | return (rc); |
| 1744 | } |
| 1745 | |
| 1746 | void |
| 1747 | ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len, |
| 1748 | u_int32_t flags) |
| 1749 | { |
| 1750 | flags |= len - 1; |
| 1751 | |
| 1752 | htolem64(&prd->dba, addr)(*(__uint64_t *)(&prd->dba) = ((__uint64_t)(addr))); |
| 1753 | htolem32(&prd->flags, flags)(*(__uint32_t *)(&prd->flags) = ((__uint32_t)(flags))); |
| 1754 | } |
| 1755 | |
| 1756 | int |
| 1757 | ahci_load_prdt(struct ahci_ccb *ccb) |
| 1758 | { |
| 1759 | struct ahci_port *ap = ccb->ccb_port; |
| 1760 | struct ahci_softc *sc = ap->ap_sc; |
| 1761 | struct ata_xfer *xa = &ccb->ccb_xa; |
| 1762 | struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; |
| 1763 | bus_dmamap_t dmap = ccb->ccb_dmamap; |
| 1764 | struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; |
| 1765 | int i, error; |
| 1766 | |
| 1767 | if (xa->datalen == 0) { |
| 1768 | ccb->ccb_cmd_hdr->prdtl = 0; |
| 1769 | return (0); |
| 1770 | } |
| 1771 | |
| 1772 | error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap) , (xa->data), (xa->datalen), (((void *)0)), ((xa->flags & (1<<2)) ? 0x0001 : 0x0000)) |
| 1773 | (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK)(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap) , (xa->data), (xa->datalen), (((void *)0)), ((xa->flags & (1<<2)) ? 0x0001 : 0x0000)); |
| 1774 | if (error != 0) { |
| 1775 | printf("%s: error %d loading dmamap\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), error); |
| 1776 | return (1); |
| 1777 | } |
| 1778 | |
| 1779 | for (i = 0; i < dmap->dm_nsegs - 1; i++) { |
| 1780 | ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr, |
| 1781 | dmap->dm_segs[i].ds_len, 0); |
| 1782 | } |
| 1783 | |
| 1784 | ahci_load_prdt_seg(&prdt[i], |
| 1785 | dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len, |
| 1786 | ISSET(xa->flags, ATA_F_PIO)((xa->flags) & ((1<<4))) ? AHCI_PRDT_FLAG_INTR(1<<31) : 0); |
| 1787 | |
| 1788 | htolem16(&cmd_slot->prdtl, dmap->dm_nsegs)(*(__uint16_t *)(&cmd_slot->prdtl) = ((__uint16_t)(dmap ->dm_nsegs))); |
| 1789 | |
| 1790 | bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04)) |
| 1791 | (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04)) |
| 1792 | BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04)); |
| 1793 | |
| 1794 | return (0); |
| 1795 | } |
| 1796 | |
| 1797 | void |
| 1798 | ahci_unload_prdt(struct ahci_ccb *ccb) |
| 1799 | { |
| 1800 | struct ahci_port *ap = ccb->ccb_port; |
| 1801 | struct ahci_softc *sc = ap->ap_sc; |
| 1802 | struct ata_xfer *xa = &ccb->ccb_xa; |
| 1803 | bus_dmamap_t dmap = ccb->ccb_dmamap; |
| 1804 | |
| 1805 | if (xa->datalen != 0) { |
| 1806 | bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08)) |
| 1807 | (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08)) |
| 1808 | BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08)); |
| 1809 | |
| 1810 | bus_dmamap_unload(sc->sc_dmat, dmap)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (dmap )); |
| 1811 | |
| 1812 | if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) |
| 1813 | xa->resid = 0; |
| 1814 | else |
| 1815 | xa->resid = xa->datalen - |
| 1816 | lemtoh32(&ccb->ccb_cmd_hdr->prdbc)((__uint32_t)(*(__uint32_t *)(&ccb->ccb_cmd_hdr->prdbc ))); |
| 1817 | } |
| 1818 | } |
| 1819 | |
| 1820 | int |
| 1821 | ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) |
| 1822 | { |
| 1823 | struct ahci_port *ap = ccb->ccb_port; |
| 1824 | int s; |
| 1825 | |
| 1826 | s = splbio()splraise(0x3); |
| 1827 | ahci_start(ccb); |
| 1828 | do { |
| 1829 | if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot )) |
| 1830 | 1 << ccb->ccb_slot)((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot ))) { |
| 1831 | splx(s)spllower(s); |
| 1832 | return (0); |
| 1833 | } |
| 1834 | if (ccb->ccb_xa.state == ATA_S_ERROR3) { |
| 1835 | DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n", |
| 1836 | PORTNAME(ap), ccb->ccb_slot); |
| 1837 | /* pretend it timed out? */ |
| 1838 | if (timeout_fn != NULL((void *)0)) { |
| 1839 | timeout_fn(ccb); |
| 1840 | } |
| 1841 | splx(s)spllower(s); |
| 1842 | return (1); |
| 1843 | } |
| 1844 | |
| 1845 | delay(1000)(*delay_func)(1000); |
| 1846 | } while (--timeout > 0); |
| 1847 | |
| 1848 | /* Run timeout while at splbio, otherwise ahci_intr could interfere. */ |
| 1849 | if (timeout_fn != NULL((void *)0)) |
| 1850 | timeout_fn(ccb); |
| 1851 | |
| 1852 | splx(s)spllower(s); |
| 1853 | |
| 1854 | return (1); |
| 1855 | } |
| 1856 | |
| 1857 | void |
| 1858 | ahci_start(struct ahci_ccb *ccb) |
| 1859 | { |
| 1860 | struct ahci_port *ap = ccb->ccb_port; |
| 1861 | struct ahci_softc *sc = ap->ap_sc; |
| 1862 | |
| 1863 | /* Zero transferred byte count before transfer */ |
| 1864 | ccb->ccb_cmd_hdr->prdbc = 0; |
| 1865 | |
| 1866 | /* Sync command list entry and corresponding command table entry */ |
| 1867 | bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) ) |
| 1868 | ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) ) |
| 1869 | sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) ); |
| 1870 | bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 )) |
| 1871 | ccb->ccb_slot * sizeof(struct ahci_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 )) |
| 1872 | sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 )); |
| 1873 | |
| 1874 | /* Prepare RFIS area for write by controller */ |
| 1875 | bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x01)) |
| 1876 | sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x01)); |
| 1877 | |
| 1878 | if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) { |
| 1879 | /* Issue NCQ commands only when there are no outstanding |
| 1880 | * standard commands. */ |
| 1881 | if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending)(((&ap->ap_ccb_pending)->tqh_first) == ((void *)0)) || |
| 1882 | (ap->ap_sactive != 0 && |
| 1883 | ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) { |
| 1884 | TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_pending)->tqh_last; *(& ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0); |
| 1885 | } else { |
| 1886 | KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1886, "ap->ap_active_cnt == 0" )); |
| 1887 | ap->ap_sactive |= (1 << ccb->ccb_slot); |
| 1888 | ccb->ccb_xa.state = ATA_S_ONCHIP5; |
| 1889 | ahci_pwrite(ap, AHCI_PREG_SACT0x34, 1 << ccb->ccb_slot); |
| 1890 | ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot); |
| 1891 | ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port; |
| 1892 | } |
| 1893 | } else { |
| 1894 | /* Wait for all NCQ commands to finish before issuing standard |
| 1895 | * command. */ |
| 1896 | if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) |
| 1897 | TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_pending)->tqh_last; *(& ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0); |
| 1898 | else if (ap->ap_active_cnt < 2) { |
| 1899 | ap->ap_active |= 1 << ccb->ccb_slot; |
| 1900 | ccb->ccb_xa.state = ATA_S_ONCHIP5; |
| 1901 | ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot); |
| 1902 | ap->ap_active_cnt++; |
| 1903 | } |
| 1904 | } |
| 1905 | } |
| 1906 | |
| 1907 | void |
| 1908 | ahci_issue_pending_ncq_commands(struct ahci_port *ap) |
| 1909 | { |
| 1910 | struct ahci_ccb *nextccb; |
| 1911 | u_int32_t sact_change = 0; |
| 1912 | |
| 1913 | KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1913, "ap->ap_active_cnt == 0" )); |
| 1914 | |
| 1915 | nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first); |
| 1916 | if (nextccb == NULL((void *)0) || !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6))) |
| 1917 | return; |
| 1918 | |
| 1919 | /* Start all the NCQ commands at the head of the pending list. |
| 1920 | * If a port multiplier is attached to the port, we can only |
| 1921 | * issue commands for one of its ports at a time. |
| 1922 | */ |
| 1923 | if (ap->ap_sactive != 0 && |
| 1924 | ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) { |
| 1925 | return; |
| 1926 | } |
| 1927 | |
| 1928 | ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port; |
| 1929 | do { |
| 1930 | TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void *)0)) ( nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb )->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)-> tqh_last = (nextccb)->ccb_entry.tqe_prev; *(nextccb)->ccb_entry .tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)-> ccb_entry.tqe_prev) = ((void *)-1); ((nextccb)->ccb_entry. tqe_next) = ((void *)-1); } while (0); |
| 1931 | sact_change |= 1 << nextccb->ccb_slot; |
| 1932 | nextccb->ccb_xa.state = ATA_S_ONCHIP5; |
| 1933 | nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first); |
| 1934 | } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) && |
| 1935 | (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port)); |
| 1936 | |
| 1937 | ap->ap_sactive |= sact_change; |
| 1938 | ahci_pwrite(ap, AHCI_PREG_SACT0x34, sact_change); |
| 1939 | ahci_pwrite(ap, AHCI_PREG_CI0x38, sact_change); |
| 1940 | } |
| 1941 | |
| 1942 | void |
| 1943 | ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) |
| 1944 | { |
| 1945 | struct ahci_ccb *nextccb; |
| 1946 | |
| 1947 | nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first); |
| 1948 | if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6))) { |
| 1949 | if (last_was_ncq) { |
| 1950 | KASSERT(nextccb->ccb_xa.pmp_port !=((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void )0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951 , "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port")) |
| 1951 | ap->ap_pmp_ncq_port)((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void )0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951 , "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port")); |
| 1952 | /* otherwise it should have been started already */ |
| 1953 | } else { |
| 1954 | ap->ap_active_cnt--; |
| 1955 | } |
| 1956 | |
| 1957 | /* Issue NCQ commands only when there are no outstanding |
| 1958 | * standard commands, and previous NCQ commands for other |
| 1959 | * PMP ports have finished. |
| 1960 | */ |
| 1961 | if (ap->ap_active == 0) |
| 1962 | ahci_issue_pending_ncq_commands(ap); |
| 1963 | else |
| 1964 | KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1964, "ap->ap_active_cnt == 1" )); |
| 1965 | } else if (nextccb) { |
| 1966 | if (ap->ap_sactive != 0 || last_was_ncq) |
| 1967 | KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1967, "ap->ap_active_cnt == 0" )); |
| 1968 | |
| 1969 | /* Wait for all NCQ commands to finish before issuing standard |
| 1970 | * command. */ |
| 1971 | if (ap->ap_sactive != 0) |
| 1972 | return; |
| 1973 | |
| 1974 | /* Keep up to 2 standard commands on-chip at a time. */ |
| 1975 | do { |
| 1976 | TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void *)0)) ( nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb )->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)-> tqh_last = (nextccb)->ccb_entry.tqe_prev; *(nextccb)->ccb_entry .tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)-> ccb_entry.tqe_prev) = ((void *)-1); ((nextccb)->ccb_entry. tqe_next) = ((void *)-1); } while (0); |
| 1977 | ap->ap_active |= 1 << nextccb->ccb_slot; |
| 1978 | nextccb->ccb_xa.state = ATA_S_ONCHIP5; |
| 1979 | ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << nextccb->ccb_slot); |
| 1980 | if (last_was_ncq) |
| 1981 | ap->ap_active_cnt++; |
| 1982 | if (ap->ap_active_cnt == 2) |
| 1983 | break; |
| 1984 | KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1984, "ap->ap_active_cnt == 1" )); |
| 1985 | nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first); |
| 1986 | } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6))); |
| 1987 | } else if (!last_was_ncq) { |
| 1988 | KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2)((ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2) ? ( void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1988, "ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2" )); |
| 1989 | |
| 1990 | /* Standard command finished, none waiting to start. */ |
| 1991 | ap->ap_active_cnt--; |
| 1992 | } else { |
| 1993 | KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1993, "ap->ap_active_cnt == 0" )); |
| 1994 | |
| 1995 | /* NCQ command finished. */ |
| 1996 | } |
| 1997 | } |
| 1998 | |
| 1999 | int |
| 2000 | ahci_intr(void *arg) |
| 2001 | { |
| 2002 | struct ahci_softc *sc = arg; |
| 2003 | u_int32_t is, ack = 0; |
| 2004 | int port; |
| 2005 | |
| 2006 | /* Read global interrupt status */ |
| 2007 | is = ahci_read(sc, AHCI_REG_IS0x008); |
| 2008 | if (is == 0 || is == 0xffffffff) |
| 2009 | return (0); |
| 2010 | ack = is; |
| 2011 | |
| 2012 | #ifdef AHCI_COALESCE |
| 2013 | /* Check coalescing interrupt first */ |
| 2014 | if (is & sc->sc_ccc_mask) { |
| 2015 | DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", |
| 2016 | DEVNAME(sc)); |
| 2017 | is &= ~sc->sc_ccc_mask; |
| 2018 | is |= sc->sc_ccc_ports_cur; |
| 2019 | } |
| 2020 | #endif |
| 2021 | |
| 2022 | /* Process interrupts for each port */ |
| 2023 | while (is) { |
| 2024 | port = ffs(is) - 1; |
| 2025 | if (sc->sc_ports[port]) |
| 2026 | ahci_port_intr(sc->sc_ports[port], |
| 2027 | AHCI_PREG_CI_ALL_SLOTS0xffffffff); |
| 2028 | is &= ~(1 << port); |
| 2029 | } |
| 2030 | |
| 2031 | /* Finally, acknowledge global interrupt */ |
| 2032 | ahci_write(sc, AHCI_REG_IS0x008, ack); |
| 2033 | |
| 2034 | return (1); |
| 2035 | } |
| 2036 | |
| 2037 | u_int32_t |
| 2038 | ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) |
| 2039 | { |
| 2040 | struct ahci_softc *sc = ap->ap_sc; |
| 2041 | u_int32_t is, ci_saved, ci_masked, processed = 0; |
| 2042 | int slot, need_restart = 0; |
| 2043 | int process_error = 0; |
| 2044 | struct ahci_ccb *ccb; |
| 2045 | volatile u_int32_t *active; |
| 2046 | #ifdef DIAGNOSTIC1 |
| 2047 | u_int32_t tmp; |
| 2048 | #endif |
| 2049 | |
| 2050 | is = ahci_pread(ap, AHCI_PREG_IS0x10); |
| 2051 | |
| 2052 | /* Ack port interrupt only if checking all command slots. */ |
| 2053 | if (ci_mask == AHCI_PREG_CI_ALL_SLOTS0xffffffff) |
| 2054 | ahci_pwrite(ap, AHCI_PREG_IS0x10, is); |
| 2055 | |
| 2056 | if (is) |
| 2057 | DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), |
| 2058 | is, AHCI_PFMT_IS); |
| 2059 | |
| 2060 | if (ap->ap_sactive) { |
| 2061 | /* Active NCQ commands - use SActive instead of CI */ |
| 2062 | KASSERT(ap->ap_active == 0)((ap->ap_active == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2062, "ap->ap_active == 0")); |
| 2063 | KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2063, "ap->ap_active_cnt == 0" )); |
| 2064 | ci_saved = ahci_pread(ap, AHCI_PREG_SACT0x34); |
| 2065 | active = &ap->ap_sactive; |
| 2066 | } else { |
| 2067 | /* Save CI */ |
| 2068 | ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38); |
| 2069 | active = &ap->ap_active; |
| 2070 | } |
| 2071 | |
| 2072 | if (is & AHCI_PREG_IS_TFES(1<<30)) { |
| 2073 | process_error = 1; |
| 2074 | } else if (is & AHCI_PREG_IS_DHRS(1<<0)) { |
| 2075 | u_int32_t tfd; |
| 2076 | u_int32_t cmd; |
| 2077 | u_int32_t serr; |
| 2078 | |
| 2079 | tfd = ahci_pread(ap, AHCI_PREG_TFD0x20); |
| 2080 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18); |
| 2081 | serr = ahci_pread(ap, AHCI_PREG_SERR0x30); |
| 2082 | if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) && |
| 2083 | (cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) { |
| 2084 | DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:" |
| 2085 | " %b, DIAG: %b\n", PORTNAME(ap), tfd, |
| 2086 | AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), |
| 2087 | AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), |
| 2088 | AHCI_PFMT_SERR_DIAG); |
| 2089 | process_error = 1; |
| 2090 | } else { |
| 2091 | /* rfis copy back is in the normal execution path */ |
| 2092 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0)); |
| 2093 | } |
| 2094 | } |
| 2095 | |
| 2096 | /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ |
| 2097 | if (process_error) { |
| 2098 | u_int32_t tfd, serr; |
| 2099 | int err_slot; |
| 2100 | |
| 2101 | tfd = ahci_pread(ap, AHCI_PREG_TFD0x20); |
| 2102 | serr = ahci_pread(ap, AHCI_PREG_SERR0x30); |
| 2103 | |
| 2104 | if (ap->ap_sactive == 0) { |
| 2105 | /* Errored slot is easy to determine from CMD. */ |
| 2106 | err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,(((ahci_pread(ap, 0x18)) >> 8) & 0x1f) |
| 2107 | AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f); |
| 2108 | |
| 2109 | if ((ci_saved & (1 << err_slot)) == 0) { |
| 2110 | /* |
| 2111 | * Hardware doesn't seem to report correct |
| 2112 | * slot number. If there's only one |
| 2113 | * outstanding command we can cope, |
| 2114 | * otherwise fail all active commands. |
| 2115 | */ |
| 2116 | if (ap->ap_active_cnt == 1) |
| 2117 | err_slot = ffs(ap->ap_active) - 1; |
| 2118 | else |
| 2119 | goto failall; |
| 2120 | } |
| 2121 | |
| 2122 | ccb = &ap->ap_ccbs[err_slot]; |
| 2123 | |
| 2124 | /* Preserve received taskfile data from the RFIS. */ |
| 2125 | memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h))) |
| 2126 | sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h))); |
| 2127 | } else |
| 2128 | err_slot = -1; /* Must extract error from log page */ |
| 2129 | |
| 2130 | DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" |
| 2131 | " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, |
| 2132 | AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), |
| 2133 | AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), |
| 2134 | AHCI_PFMT_SERR_DIAG); |
| 2135 | |
| 2136 | /* Turn off ST to clear CI and SACT. */ |
| 2137 | ahci_port_stop(ap, 0); |
| 2138 | need_restart = 1; |
| 2139 | |
| 2140 | /* Clear SERR to enable capturing new errors. */ |
| 2141 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, serr); |
| 2142 | |
| 2143 | /* Acknowledge the interrupts we can recover from. */ |
| 2144 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_TFES(1<<30) | |
| 2145 | AHCI_PREG_IS_IFS(1<<27)); |
| 2146 | is = ahci_pread(ap, AHCI_PREG_IS0x10); |
| 2147 | |
| 2148 | /* If device hasn't cleared its busy status, try to idle it. */ |
| 2149 | if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)((tfd) & ((1<<7) | (1<<3)))) { |
| 2150 | |
| 2151 | if ((ap->ap_state == AP_S_PMP_PORT_PROBE2) || |
| 2152 | (ap->ap_state == AP_S_ERROR_RECOVERY3)) { |
| 2153 | /* can't reset the port here, just make sure |
| 2154 | * the operation fails and the port still works. |
| 2155 | */ |
| 2156 | } else if (ap->ap_pmp_ports != 0 && err_slot != -1) { |
| 2157 | printf("%s: error on PMP port %d, idling " |
| 2158 | "device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 2159 | ccb->ccb_xa.pmp_port); |
| 2160 | if (ahci_pmp_port_softreset(ap, |
| 2161 | ccb->ccb_xa.pmp_port) == 0) { |
| 2162 | printf("%s: unable to softreset port " |
| 2163 | "%d\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 2164 | ccb->ccb_xa.pmp_port); |
| 2165 | if (ahci_pmp_port_portreset(ap, |
| 2166 | ccb->ccb_xa.pmp_port)) { |
| 2167 | printf("%s: failed to port " |
| 2168 | " reset %d, giving up on " |
| 2169 | "it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), |
| 2170 | ccb->ccb_xa.pmp_port); |
| 2171 | goto fatal; |
| 2172 | } |
| 2173 | } |
| 2174 | } else { |
| 2175 | printf("%s: attempting to idle device\n", |
| 2176 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 2177 | if (ahci_port_softreset(ap)) { |
| 2178 | printf("%s: failed to soft reset " |
| 2179 | "device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 2180 | if (ahci_port_portreset(ap, 0)) { |
| 2181 | printf("%s: failed to port " |
| 2182 | "reset device, give up on " |
| 2183 | "it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 2184 | goto fatal; |
| 2185 | } |
| 2186 | } |
| 2187 | } |
| 2188 | |
| 2189 | /* Had to reset device, can't gather extended info. */ |
| 2190 | } else if (ap->ap_sactive) { |
| 2191 | /* Recover the NCQ error from log page 10h. |
| 2192 | * We can only have queued commands active for one port |
| 2193 | * at a time, so we know which device errored. |
| 2194 | */ |
| 2195 | ahci_port_read_ncq_error(ap, &err_slot, |
| 2196 | ap->ap_pmp_ncq_port); |
| 2197 | if (err_slot < 0) |
| 2198 | goto failall; |
| 2199 | |
| 2200 | DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", |
| 2201 | PORTNAME(ap), err_slot); |
| 2202 | |
| 2203 | ccb = &ap->ap_ccbs[err_slot]; |
| 2204 | if (ccb->ccb_xa.state != ATA_S_ONCHIP5) { |
| 2205 | printf("%s: NCQ errored slot %d is idle" |
| 2206 | " (%08x active)\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), err_slot, |
| 2207 | ci_saved); |
| 2208 | goto failall; |
| 2209 | } |
| 2210 | } else { |
| 2211 | /* Didn't reset, could gather extended info from log. */ |
| 2212 | } |
| 2213 | |
| 2214 | /* |
| 2215 | * If we couldn't determine the errored slot, reset the port |
| 2216 | * and fail all the active slots. |
| 2217 | */ |
| 2218 | if (err_slot == -1) { |
| 2219 | if (ahci_port_softreset(ap) != 0 && |
| 2220 | ahci_port_portreset(ap, 0) != 0) { |
| 2221 | printf("%s: couldn't reset after NCQ error, " |
| 2222 | "disabling device.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 2223 | goto fatal; |
| 2224 | } |
| 2225 | printf("%s: couldn't recover NCQ error, failing " |
| 2226 | "all outstanding commands.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 2227 | goto failall; |
| 2228 | } |
| 2229 | |
| 2230 | /* Clear the failed command in saved CI so completion runs. */ |
| 2231 | ci_saved &= ~(1 << err_slot); |
| 2232 | |
| 2233 | /* Note the error in the ata_xfer. */ |
| 2234 | KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2234, "ccb->ccb_xa.state == ATA_S_ONCHIP" )); |
| 2235 | ccb->ccb_xa.state = ATA_S_ERROR3; |
| 2236 | |
| 2237 | #ifdef DIAGNOSTIC1 |
| 2238 | /* There may only be one outstanding standard command now. */ |
| 2239 | if (ap->ap_sactive == 0) { |
| 2240 | tmp = ci_saved; |
| 2241 | if (tmp) { |
| 2242 | slot = ffs(tmp) - 1; |
| 2243 | tmp &= ~(1 << slot); |
| 2244 | KASSERT(tmp == 0)((tmp == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2244, "tmp == 0")); |
| 2245 | } |
| 2246 | } |
| 2247 | #endif |
| 2248 | } |
| 2249 | |
| 2250 | /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts |
| 2251 | * when there's a normal SATA device attached. |
| 2252 | */ |
| 2253 | if ((ap->ap_state == AP_S_PMP_PROBE1) && |
| 2254 | (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE(1<<1)) && |
| 2255 | (is & AHCI_PREG_IS_IPMS(1<<23))) { |
| 2256 | slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f); |
| 2257 | DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n", |
| 2258 | PORTNAME(ap), slot); |
| 2259 | |
| 2260 | ccb = &ap->ap_ccbs[slot]; |
| 2261 | ccb->ccb_xa.state = ATA_S_ERROR3; |
| 2262 | |
| 2263 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IPMS(1<<23)); |
| 2264 | is &= ~AHCI_PREG_IS_IPMS(1<<23); |
| 2265 | } |
| 2266 | |
| 2267 | /* ignore IFS errors while resetting a PMP port */ |
| 2268 | if ((is & AHCI_PREG_IS_IFS(1<<27)) /*&& ap->ap_pmp_ignore_ifs*/) { |
| 2269 | DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP " |
| 2270 | "port\n", PORTNAME(ap)); |
| 2271 | |
| 2272 | need_restart = 1; |
| 2273 | ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1); |
| 2274 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27)); |
| 2275 | is &= ~AHCI_PREG_IS_IFS(1<<27); |
| 2276 | goto failall; |
| 2277 | } |
| 2278 | |
| 2279 | /* Check for remaining errors - they are fatal. */ |
| 2280 | if (is & (AHCI_PREG_IS_TFES(1<<30) | AHCI_PREG_IS_HBFS(1<<29) | AHCI_PREG_IS_IFS(1<<27) | |
| 2281 | AHCI_PREG_IS_OFS(1<<24) | AHCI_PREG_IS_UFS(1<<4))) { |
| 2282 | printf("%s: unrecoverable errors (IS: %b), disabling port.\n", |
| 2283 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), is, AHCI_PFMT_IS"\20" "\040CPDS" "\037TFES" "\036HBFS" "\035HBDS" "\034IFS" "\033INFS" "\031OFS" "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" "\007PCS" "\005UFS" "\004SDBS" "\003DSS" "\002PSS" "\001DHRS"); |
| 2284 | |
| 2285 | /* XXX try recovery first */ |
| 2286 | goto fatal; |
| 2287 | } |
| 2288 | |
| 2289 | /* Fail all outstanding commands if we know the port won't recover. */ |
| 2290 | if (ap->ap_state == AP_S_FATAL_ERROR4) { |
| 2291 | fatal: |
| 2292 | ap->ap_state = AP_S_FATAL_ERROR4; |
| 2293 | failall: |
| 2294 | |
| 2295 | /* Ensure port is shut down. */ |
| 2296 | ahci_port_stop(ap, 1); |
| 2297 | |
| 2298 | /* Error all the active slots. */ |
| 2299 | ci_masked = ci_saved & *active; |
| 2300 | while (ci_masked) { |
| 2301 | slot = ffs(ci_masked) - 1; |
| 2302 | ccb = &ap->ap_ccbs[slot]; |
| 2303 | ci_masked &= ~(1 << slot); |
| 2304 | ccb->ccb_xa.state = ATA_S_ERROR3; |
| 2305 | } |
| 2306 | |
| 2307 | /* Run completion for all active slots. */ |
| 2308 | ci_saved &= ~*active; |
| 2309 | |
| 2310 | /* Don't restart the port if our problems were deemed fatal. */ |
| 2311 | if (ap->ap_state == AP_S_FATAL_ERROR4) |
| 2312 | need_restart = 0; |
| 2313 | } |
| 2314 | |
| 2315 | /* |
| 2316 | * CCB completion is detected by noticing its slot's bit in CI has |
| 2317 | * changed to zero some time after we activated it. |
| 2318 | * If we are polling, we may only be interested in particular slot(s). |
| 2319 | */ |
| 2320 | ci_masked = ~ci_saved & *active & ci_mask; |
| 2321 | while (ci_masked) { |
| 2322 | slot = ffs(ci_masked) - 1; |
| 2323 | ccb = &ap->ap_ccbs[slot]; |
| 2324 | ci_masked &= ~(1 << slot); |
| 2325 | |
| 2326 | DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", |
| 2327 | PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? |
| 2328 | " (error)" : ""); |
| 2329 | |
| 2330 | bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) ) |
| 2331 | AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) ) |
| 2332 | ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) ) |
| 2333 | sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) ); |
| 2334 | |
| 2335 | bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 )) |
| 2336 | AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 )) |
| 2337 | ccb->ccb_slot * sizeof(struct ahci_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 )) |
| 2338 | sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 )); |
| 2339 | |
| 2340 | bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02)) |
| 2341 | AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02)) |
| 2342 | sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02)); |
| 2343 | |
| 2344 | *active &= ~(1 << ccb->ccb_slot); |
| 2345 | /* Copy the rfis into the ccb if we were asked for it */ |
| 2346 | if (ccb->ccb_xa.state == ATA_S_ONCHIP5 && |
| 2347 | ccb->ccb_xa.flags & ATA_F_GET_RFIS(1<<8)) { |
| 2348 | memcpy(&ccb->ccb_xa.rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h))) |
| 2349 | ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h))) |
| 2350 | sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h))); |
| 2351 | } |
| 2352 | |
| 2353 | ccb->ccb_done(ccb); |
| 2354 | |
| 2355 | processed |= 1 << ccb->ccb_slot; |
| 2356 | } |
| 2357 | |
| 2358 | if (need_restart) { |
| 2359 | /* Restart command DMA on the port */ |
| 2360 | ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0))); |
| 2361 | |
| 2362 | /* Re-enable outstanding commands on port. */ |
| 2363 | if (ci_saved) { |
| 2364 | #ifdef DIAGNOSTIC1 |
| 2365 | tmp = ci_saved; |
| 2366 | while (tmp) { |
| 2367 | slot = ffs(tmp) - 1; |
| 2368 | tmp &= ~(1 << slot); |
| 2369 | ccb = &ap->ap_ccbs[slot]; |
| 2370 | KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2370, "ccb->ccb_xa.state == ATA_S_ONCHIP" )); |
| 2371 | KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap-> ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)" )) |
| 2372 | (!!ap->ap_sactive))(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap-> ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)" )); |
| 2373 | } |
| 2374 | #endif |
| 2375 | DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " |
| 2376 | "re-enabling%s slots %08x\n", PORTNAME(ap), |
| 2377 | ap->ap_sactive ? " NCQ" : "", ci_saved); |
| 2378 | |
| 2379 | if (ap->ap_sactive) |
| 2380 | ahci_pwrite(ap, AHCI_PREG_SACT0x34, ci_saved); |
| 2381 | ahci_pwrite(ap, AHCI_PREG_CI0x38, ci_saved); |
| 2382 | } |
| 2383 | } |
| 2384 | |
| 2385 | return (processed); |
| 2386 | } |
| 2387 | |
| 2388 | struct ahci_ccb * |
| 2389 | ahci_get_ccb(struct ahci_port *ap) |
| 2390 | { |
| 2391 | struct ahci_ccb *ccb; |
| 2392 | |
| 2393 | mtx_enter(&ap->ap_ccb_mtx); |
| 2394 | ccb = TAILQ_FIRST(&ap->ap_ccb_free)((&ap->ap_ccb_free)->tqh_first); |
| 2395 | if (ccb != NULL((void *)0)) { |
| 2396 | KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2396, "ccb->ccb_xa.state == ATA_S_PUT" )); |
| 2397 | TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb )->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = (ccb )->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = (( void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1); } while (0); |
| 2398 | ccb->ccb_xa.state = ATA_S_SETUP0; |
| 2399 | } |
| 2400 | mtx_leave(&ap->ap_ccb_mtx); |
| 2401 | |
| 2402 | return (ccb); |
| 2403 | } |
| 2404 | |
| 2405 | void |
| 2406 | ahci_put_ccb(struct ahci_ccb *ccb) |
| 2407 | { |
| 2408 | struct ahci_port *ap = ccb->ccb_port; |
| 2409 | |
| 2410 | #ifdef DIAGNOSTIC1 |
| 2411 | if (ccb->ccb_xa.state != ATA_S_COMPLETE2 && |
| 2412 | ccb->ccb_xa.state != ATA_S_TIMEOUT4 && |
| 2413 | ccb->ccb_xa.state != ATA_S_ERROR3) { |
| 2414 | printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " |
| 2415 | "slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state, |
| 2416 | ccb->ccb_slot); |
| 2417 | } |
| 2418 | #endif |
| 2419 | |
| 2420 | ccb->ccb_xa.state = ATA_S_PUT6; |
| 2421 | mtx_enter(&ap->ap_ccb_mtx); |
| 2422 | TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_free)->tqh_last; *(&ap ->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0); |
| 2423 | mtx_leave(&ap->ap_ccb_mtx); |
| 2424 | } |
| 2425 | |
| 2426 | struct ahci_ccb * |
| 2427 | ahci_get_err_ccb(struct ahci_port *ap) |
| 2428 | { |
| 2429 | struct ahci_ccb *err_ccb; |
| 2430 | u_int32_t sact; |
| 2431 | |
| 2432 | splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__ ); } } while (0); |
| 2433 | |
| 2434 | /* No commands may be active on the chip. */ |
| 2435 | sact = ahci_pread(ap, AHCI_PREG_SACT0x34); |
| 2436 | if (sact != 0) |
| 2437 | printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); |
| 2438 | KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2438, "ahci_pread(ap, AHCI_PREG_CI) == 0" )); |
| 2439 | |
| 2440 | #ifdef DIAGNOSTIC1 |
| 2441 | KASSERT(ap->ap_err_busy == 0)((ap->ap_err_busy == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2441, "ap->ap_err_busy == 0" )); |
| 2442 | ap->ap_err_busy = 1; |
| 2443 | #endif |
| 2444 | /* Save outstanding command state. */ |
| 2445 | ap->ap_err_saved_active = ap->ap_active; |
| 2446 | ap->ap_err_saved_active_cnt = ap->ap_active_cnt; |
| 2447 | ap->ap_err_saved_sactive = ap->ap_sactive; |
| 2448 | |
| 2449 | /* |
| 2450 | * Pretend we have no commands outstanding, so that completions won't |
| 2451 | * run prematurely. |
| 2452 | */ |
| 2453 | ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; |
| 2454 | |
| 2455 | /* |
| 2456 | * Grab a CCB to use for error recovery. This should never fail, as |
| 2457 | * we ask atascsi to reserve one for us at init time. |
| 2458 | */ |
| 2459 | err_ccb = ap->ap_ccb_err; |
| 2460 | err_ccb->ccb_xa.flags = 0; |
| 2461 | err_ccb->ccb_xa.state = ATA_S_SETUP0; |
| 2462 | err_ccb->ccb_done = ahci_empty_done; |
| 2463 | |
| 2464 | return (err_ccb); |
| 2465 | } |
| 2466 | |
| 2467 | void |
| 2468 | ahci_put_err_ccb(struct ahci_ccb *ccb) |
| 2469 | { |
| 2470 | struct ahci_port *ap = ccb->ccb_port; |
| 2471 | u_int32_t sact; |
| 2472 | |
| 2473 | splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__ ); } } while (0); |
| 2474 | |
| 2475 | #ifdef DIAGNOSTIC1 |
| 2476 | KASSERT(ap->ap_err_busy)((ap->ap_err_busy) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2476, "ap->ap_err_busy")); |
| 2477 | #endif |
| 2478 | /* No commands may be active on the chip */ |
| 2479 | sact = ahci_pread(ap, AHCI_PREG_SACT0x34); |
| 2480 | if (sact != 0) |
| 2481 | printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact); |
| 2482 | KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2482, "ahci_pread(ap, AHCI_PREG_CI) == 0" )); |
| 2483 | |
| 2484 | /* Done with the CCB */ |
| 2485 | KASSERT(ccb == ap->ap_ccb_err)((ccb == ap->ap_ccb_err) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2485, "ccb == ap->ap_ccb_err" )); |
| 2486 | |
| 2487 | /* Restore outstanding command state */ |
| 2488 | ap->ap_sactive = ap->ap_err_saved_sactive; |
| 2489 | ap->ap_active_cnt = ap->ap_err_saved_active_cnt; |
| 2490 | ap->ap_active = ap->ap_err_saved_active; |
| 2491 | |
| 2492 | #ifdef DIAGNOSTIC1 |
| 2493 | ap->ap_err_busy = 0; |
| 2494 | #endif |
| 2495 | } |
| 2496 | |
| 2497 | struct ahci_ccb * |
| 2498 | ahci_get_pmp_ccb(struct ahci_port *ap) |
| 2499 | { |
| 2500 | struct ahci_ccb *ccb; |
| 2501 | u_int32_t sact; |
| 2502 | |
| 2503 | /* some PMP commands need to be issued on slot 1, |
| 2504 | * particularly the command that clears SRST and |
| 2505 | * fetches the device signature. |
| 2506 | * |
| 2507 | * ensure the chip is idle and ccb 1 is available. |
| 2508 | */ |
| 2509 | splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__ ); } } while (0); |
| 2510 | |
| 2511 | sact = ahci_pread(ap, AHCI_PREG_SACT0x34); |
| 2512 | if (sact != 0) |
| 2513 | printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact); |
| 2514 | KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2514, "ahci_pread(ap, AHCI_PREG_CI) == 0" )); |
| 2515 | |
| 2516 | ccb = &ap->ap_ccbs[1]; |
| 2517 | KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2517, "ccb->ccb_xa.state == ATA_S_PUT" )); |
| 2518 | ccb->ccb_xa.flags = 0; |
| 2519 | ccb->ccb_done = ahci_pmp_cmd_done; |
| 2520 | |
| 2521 | mtx_enter(&ap->ap_ccb_mtx); |
| 2522 | TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb )->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = (ccb )->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = (( void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1); } while (0); |
| 2523 | mtx_leave(&ap->ap_ccb_mtx); |
| 2524 | |
| 2525 | return ccb; |
| 2526 | } |
| 2527 | |
| 2528 | void |
| 2529 | ahci_put_pmp_ccb(struct ahci_ccb *ccb) |
| 2530 | { |
| 2531 | struct ahci_port *ap = ccb->ccb_port; |
| 2532 | u_int32_t sact; |
| 2533 | |
| 2534 | /* make sure this is the right ccb */ |
| 2535 | KASSERT(ccb == &ap->ap_ccbs[1])((ccb == &ap->ap_ccbs[1]) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2535, "ccb == &ap->ap_ccbs[1]" )); |
| 2536 | |
| 2537 | /* No commands may be active on the chip */ |
| 2538 | sact = ahci_pread(ap, AHCI_PREG_SACT0x34); |
| 2539 | if (sact != 0) |
| 2540 | printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact); |
| 2541 | KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2541, "ahci_pread(ap, AHCI_PREG_CI) == 0" )); |
| 2542 | |
| 2543 | ccb->ccb_xa.state = ATA_S_PUT6; |
| 2544 | mtx_enter(&ap->ap_ccb_mtx); |
| 2545 | TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_free)->tqh_last; *(&ap ->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0); |
| 2546 | mtx_leave(&ap->ap_ccb_mtx); |
| 2547 | } |
| 2548 | |
| 2549 | int |
| 2550 | ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port) |
| 2551 | { |
| 2552 | struct ahci_ccb *ccb; |
| 2553 | struct ahci_cmd_hdr *cmd_slot; |
| 2554 | u_int32_t cmd; |
| 2555 | struct ata_fis_h2d *fis; |
| 2556 | int rc = EIO5, oldstate; |
| 2557 | |
| 2558 | DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); |
| 2559 | oldstate = ap->ap_state; |
| 2560 | ap->ap_state = AP_S_ERROR_RECOVERY3; |
| 2561 | |
| 2562 | /* Save command register state. */ |
| 2563 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 2564 | |
| 2565 | /* Port should have been idled already. Start it. */ |
| 2566 | KASSERT((cmd & AHCI_PREG_CMD_CR) == 0)(((cmd & (1<<15)) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2566, "(cmd & AHCI_PREG_CMD_CR) == 0" )); |
| 2567 | ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0))); |
| 2568 | |
| 2569 | /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ |
| 2570 | ccb = ahci_get_err_ccb(ap); |
| 2571 | ccb->ccb_xa.flags = ATA_F_NOWAIT(1<<2) | ATA_F_READ(1<<0) | ATA_F_POLL(1<<3); |
| 2572 | ccb->ccb_xa.data = ap->ap_err_scratch; |
| 2573 | ccb->ccb_xa.datalen = 512; |
| 2574 | cmd_slot = ccb->ccb_cmd_hdr; |
| 2575 | memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 2576 | |
| 2577 | fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; |
| 2578 | fis->type = ATA_FIS_TYPE_H2D0x27; |
| 2579 | fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | pmp_port; |
| 2580 | fis->command = ATA_C_READ_LOG_EXT0x2f; |
| 2581 | fis->lba_low = 0x10; /* queued error log page (10h) */ |
| 2582 | fis->sector_count = 1; /* number of sectors (1) */ |
| 2583 | fis->sector_count_exp = 0; |
| 2584 | fis->lba_mid = 0; /* starting offset */ |
| 2585 | fis->lba_mid_exp = 0; |
| 2586 | fis->device = 0; |
| 2587 | |
| 2588 | htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (pmp_port << 12)))) |
| 2589 | (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 | (pmp_port << 12)))); |
| 2590 | |
| 2591 | if (ahci_load_prdt(ccb) != 0) { |
| 2592 | rc = ENOMEM12; /* XXX caller must abort all commands */ |
| 2593 | goto err; |
| 2594 | } |
| 2595 | |
| 2596 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 2597 | if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0 || |
| 2598 | ccb->ccb_xa.state == ATA_S_ERROR3) |
| 2599 | goto err; |
| 2600 | |
| 2601 | rc = 0; |
| 2602 | err: |
| 2603 | /* Abort our command, if it failed, by stopping command DMA. */ |
| 2604 | if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) { |
| 2605 | printf("%s: log page read failed, slot %d was still active.\n", |
| 2606 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot); |
| 2607 | ahci_port_stop(ap, 0); |
| 2608 | } |
| 2609 | |
| 2610 | /* Done with the error CCB now. */ |
| 2611 | ahci_unload_prdt(ccb); |
| 2612 | ahci_put_err_ccb(ccb); |
| 2613 | |
| 2614 | /* Extract failed register set and tags from the scratch space. */ |
| 2615 | if (rc == 0) { |
| 2616 | struct ata_log_page_10h *log; |
| 2617 | int err_slot; |
| 2618 | |
| 2619 | log = (struct ata_log_page_10h *)ap->ap_err_scratch; |
| 2620 | if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)((log->err_regs.type) & (0x80))) { |
| 2621 | /* Not queued bit was set - wasn't an NCQ error? */ |
| 2622 | printf("%s: read NCQ error page, but not an NCQ " |
| 2623 | "error?\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 2624 | rc = ESRCH3; |
| 2625 | } else { |
| 2626 | /* Copy back the log record as a D2H register FIS. */ |
| 2627 | *err_slotp = err_slot = log->err_regs.type & |
| 2628 | ATA_LOG_10H_TYPE_TAG_MASK0x1f; |
| 2629 | |
| 2630 | ccb = &ap->ap_ccbs[err_slot]; |
| 2631 | memcpy(&ccb->ccb_xa.rfis, &log->err_regs,__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs ), (sizeof(struct ata_fis_d2h))) |
| 2632 | sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs ), (sizeof(struct ata_fis_d2h))); |
| 2633 | ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H0x34; |
| 2634 | ccb->ccb_xa.rfis.flags = 0; |
| 2635 | } |
| 2636 | } |
| 2637 | |
| 2638 | /* Restore saved CMD register state */ |
| 2639 | ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd); |
| 2640 | ap->ap_state = oldstate; |
| 2641 | |
| 2642 | return (rc); |
| 2643 | } |
| 2644 | |
| 2645 | struct ahci_dmamem * |
| 2646 | ahci_dmamem_alloc(struct ahci_softc *sc, size_t size) |
| 2647 | { |
| 2648 | struct ahci_dmamem *adm; |
| 2649 | int nsegs; |
| 2650 | |
| 2651 | adm = malloc(sizeof(*adm), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008); |
| 2652 | if (adm == NULL((void *)0)) |
| 2653 | return (NULL((void *)0)); |
| 2654 | |
| 2655 | adm->adm_size = size; |
| 2656 | |
| 2657 | if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size ), (1), (size), (0), (0x0001 | 0x0002), (&adm->adm_map )) |
| 2658 | BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map)(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size ), (1), (size), (0), (0x0001 | 0x0002), (&adm->adm_map )) != 0) |
| 2659 | goto admfree; |
| 2660 | |
| 2661 | if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size ), ((1 << 12)), (0), (&adm->adm_seg), (1), (& nsegs), (0x0001 | 0x1000)) |
| 2662 | 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO)(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size ), ((1 << 12)), (0), (&adm->adm_seg), (1), (& nsegs), (0x0001 | 0x1000)) != 0) |
| 2663 | goto destroy; |
| 2664 | |
| 2665 | if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm ->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001 | 0x0004)) |
| 2666 | &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT)(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm ->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001 | 0x0004)) != 0) |
| 2667 | goto free; |
| 2668 | |
| 2669 | if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm-> adm_map), (adm->adm_kva), (size), (((void *)0)), (0x0001)) |
| 2670 | NULL, BUS_DMA_NOWAIT)(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm-> adm_map), (adm->adm_kva), (size), (((void *)0)), (0x0001)) != 0) |
| 2671 | goto unmap; |
| 2672 | |
| 2673 | return (adm); |
| 2674 | |
| 2675 | unmap: |
| 2676 | bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm-> adm_kva), (size)); |
| 2677 | free: |
| 2678 | bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (& adm->adm_seg), (1)); |
| 2679 | destroy: |
| 2680 | bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm ->adm_map)); |
| 2681 | admfree: |
| 2682 | free(adm, M_DEVBUF2, sizeof(*adm)); |
| 2683 | |
| 2684 | return (NULL((void *)0)); |
| 2685 | } |
| 2686 | |
| 2687 | void |
| 2688 | ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) |
| 2689 | { |
| 2690 | bus_dmamap_unload(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (adm ->adm_map)); |
| 2691 | bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm-> adm_kva), (adm->adm_size)); |
| 2692 | bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (& adm->adm_seg), (1)); |
| 2693 | bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm ->adm_map)); |
| 2694 | free(adm, M_DEVBUF2, sizeof(*adm)); |
| 2695 | } |
| 2696 | |
| 2697 | u_int32_t |
| 2698 | ahci_read(struct ahci_softc *sc, bus_size_t r) |
| 2699 | { |
| 2700 | bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, |
| 2701 | BUS_SPACE_BARRIER_READ0x01); |
| 2702 | return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)((sc->sc_iot)->read_4((sc->sc_ioh), (r)))); |
| 2703 | } |
| 2704 | |
| 2705 | void |
| 2706 | ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) |
| 2707 | { |
| 2708 | bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v)((sc->sc_iot)->write_4((sc->sc_ioh), (r), (v))); |
| 2709 | bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, |
| 2710 | BUS_SPACE_BARRIER_WRITE0x02); |
| 2711 | } |
| 2712 | |
| 2713 | int |
| 2714 | ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, |
| 2715 | u_int32_t target) |
| 2716 | { |
| 2717 | int i; |
| 2718 | |
| 2719 | for (i = 0; i < 1000; i++) { |
| 2720 | if ((ahci_read(sc, r) & mask) != target) |
| 2721 | return (0); |
| 2722 | delay(1000)(*delay_func)(1000); |
| 2723 | } |
| 2724 | |
| 2725 | return (1); |
| 2726 | } |
| 2727 | |
| 2728 | u_int32_t |
| 2729 | ahci_pread(struct ahci_port *ap, bus_size_t r) |
| 2730 | { |
| 2731 | bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, |
| 2732 | BUS_SPACE_BARRIER_READ0x01); |
| 2733 | return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)((ap->ap_sc->sc_iot)->read_4((ap->ap_ioh), (r)))); |
| 2734 | } |
| 2735 | |
| 2736 | void |
| 2737 | ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) |
| 2738 | { |
| 2739 | bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v)((ap->ap_sc->sc_iot)->write_4((ap->ap_ioh), (r), ( v))); |
| 2740 | bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, |
| 2741 | BUS_SPACE_BARRIER_WRITE0x02); |
| 2742 | } |
| 2743 | |
| 2744 | int |
| 2745 | ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, |
| 2746 | u_int32_t target, int n) |
| 2747 | { |
| 2748 | int i; |
| 2749 | |
| 2750 | for (i = 0; i < n * 1000; i++) { |
| 2751 | if ((ahci_pread(ap, r) & mask) == target) |
| 2752 | return (0); |
| 2753 | delay(1000)(*delay_func)(1000); |
| 2754 | } |
| 2755 | |
| 2756 | return (1); |
| 2757 | } |
| 2758 | |
| 2759 | int |
| 2760 | ahci_ata_probe(void *xsc, int port, int lun) |
| 2761 | { |
| 2762 | struct ahci_softc *sc = xsc; |
| 2763 | struct ahci_port *ap = sc->sc_ports[port]; |
| 2764 | |
| 2765 | if (ap == NULL((void *)0)) |
| 2766 | return (ATA_PORT_T_NONE0); |
| 2767 | |
| 2768 | if (lun != 0) { |
| 2769 | int pmp_port = lun - 1; |
| 2770 | if (pmp_port >= ap->ap_pmp_ports) { |
| 2771 | return (ATA_PORT_T_NONE0); |
| 2772 | } |
| 2773 | return (ahci_pmp_port_probe(ap, pmp_port)); |
| 2774 | } else { |
| 2775 | return (ahci_port_signature(ap)); |
| 2776 | } |
| 2777 | } |
| 2778 | |
| 2779 | void |
| 2780 | ahci_ata_free(void *xsc, int port, int lun) |
| 2781 | { |
| 2782 | |
| 2783 | } |
| 2784 | |
| 2785 | struct ata_xfer * |
| 2786 | ahci_ata_get_xfer(void *aaa_cookie, int port) |
| 2787 | { |
| 2788 | struct ahci_softc *sc = aaa_cookie; |
| 2789 | struct ahci_port *ap = sc->sc_ports[port]; |
| 2790 | struct ahci_ccb *ccb; |
| 2791 | |
| 2792 | ccb = ahci_get_ccb(ap); |
| 2793 | if (ccb == NULL((void *)0)) { |
| 2794 | DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", |
| 2795 | PORTNAME(ap)); |
| 2796 | return (NULL((void *)0)); |
| 2797 | } |
| 2798 | |
| 2799 | DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", |
| 2800 | PORTNAME(ap), ccb->ccb_slot); |
| 2801 | |
| 2802 | return ((struct ata_xfer *)ccb); |
| 2803 | } |
| 2804 | |
| 2805 | void |
| 2806 | ahci_ata_put_xfer(struct ata_xfer *xa) |
| 2807 | { |
| 2808 | struct ahci_ccb *ccb = (struct ahci_ccb *)xa; |
| 2809 | |
| 2810 | DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); |
| 2811 | |
| 2812 | ahci_put_ccb(ccb); |
| 2813 | } |
| 2814 | |
| 2815 | void |
| 2816 | ahci_ata_cmd(struct ata_xfer *xa) |
| 2817 | { |
| 2818 | struct ahci_ccb *ccb = (struct ahci_ccb *)xa; |
| 2819 | struct ahci_cmd_hdr *cmd_slot; |
| 2820 | int s; |
| 2821 | u_int16_t flags; |
| 2822 | |
| 2823 | if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR4) |
| 2824 | goto failcmd; |
| 2825 | |
| 2826 | ccb->ccb_done = ahci_ata_cmd_done; |
| 2827 | |
| 2828 | cmd_slot = ccb->ccb_cmd_hdr; |
| 2829 | flags = 5 /* FIS length (in DWORDs) */; |
| 2830 | flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12; |
| 2831 | |
| 2832 | if (xa->flags & ATA_F_WRITE(1<<1)) |
| 2833 | flags |= AHCI_CMD_LIST_FLAG_W(1<<6); |
| 2834 | |
| 2835 | if (xa->flags & ATA_F_PACKET(1<<5)) |
| 2836 | flags |= AHCI_CMD_LIST_FLAG_A(1<<5); |
| 2837 | |
| 2838 | htolem16(&cmd_slot->flags, flags)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(flags ))); |
| 2839 | |
| 2840 | if (ahci_load_prdt(ccb) != 0) |
| 2841 | goto failcmd; |
| 2842 | |
| 2843 | timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb); |
| 2844 | |
| 2845 | xa->state = ATA_S_PENDING1; |
| 2846 | |
| 2847 | if (xa->flags & ATA_F_POLL(1<<3)) |
| 2848 | ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); |
| 2849 | else { |
| 2850 | s = splbio()splraise(0x3); |
| 2851 | timeout_add_msec(&xa->stimeout, xa->timeout); |
| 2852 | ahci_start(ccb); |
| 2853 | splx(s)spllower(s); |
| 2854 | } |
| 2855 | |
| 2856 | return; |
| 2857 | |
| 2858 | failcmd: |
| 2859 | s = splbio()splraise(0x3); |
| 2860 | xa->state = ATA_S_ERROR3; |
| 2861 | ata_complete(xa); |
| 2862 | splx(s)spllower(s); |
| 2863 | } |
| 2864 | |
| 2865 | void |
| 2866 | ahci_pmp_cmd_done(struct ahci_ccb *ccb) |
| 2867 | { |
| 2868 | struct ata_xfer *xa = &ccb->ccb_xa; |
| 2869 | |
| 2870 | if (xa->state == ATA_S_ONCHIP5 || xa->state == ATA_S_ERROR3) |
| 2871 | ahci_issue_pending_commands(ccb->ccb_port, |
| 2872 | xa->flags & ATA_F_NCQ(1<<6)); |
| 2873 | |
| 2874 | xa->state = ATA_S_COMPLETE2; |
| 2875 | } |
| 2876 | |
| 2877 | |
| 2878 | void |
| 2879 | ahci_ata_cmd_done(struct ahci_ccb *ccb) |
| 2880 | { |
| 2881 | struct ata_xfer *xa = &ccb->ccb_xa; |
| 2882 | |
| 2883 | timeout_del(&xa->stimeout); |
| 2884 | |
| 2885 | if (xa->state == ATA_S_ONCHIP5 || xa->state == ATA_S_ERROR3) |
| 2886 | ahci_issue_pending_commands(ccb->ccb_port, |
| 2887 | xa->flags & ATA_F_NCQ(1<<6)); |
| 2888 | |
| 2889 | ahci_unload_prdt(ccb); |
| 2890 | |
| 2891 | if (xa->state == ATA_S_ONCHIP5) |
| 2892 | xa->state = ATA_S_COMPLETE2; |
| 2893 | #ifdef DIAGNOSTIC1 |
| 2894 | else if (xa->state != ATA_S_ERROR3 && xa->state != ATA_S_TIMEOUT4) |
| 2895 | printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " |
| 2896 | "slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), xa->state, |
| 2897 | ccb->ccb_slot); |
| 2898 | #endif |
| 2899 | if (xa->state != ATA_S_TIMEOUT4) |
| 2900 | ata_complete(xa); |
| 2901 | } |
| 2902 | |
| 2903 | void |
| 2904 | ahci_ata_cmd_timeout(void *arg) |
| 2905 | { |
| 2906 | struct ahci_ccb *ccb = arg; |
| 2907 | struct ata_xfer *xa = &ccb->ccb_xa; |
| 2908 | struct ahci_port *ap = ccb->ccb_port; |
| 2909 | int s, ccb_was_started, ncq_cmd; |
| 2910 | volatile u_int32_t *active; |
| 2911 | |
| 2912 | s = splbio()splraise(0x3); |
| 2913 | |
| 2914 | ncq_cmd = (xa->flags & ATA_F_NCQ(1<<6)); |
| 2915 | active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; |
| 2916 | |
| 2917 | if (ccb->ccb_xa.state == ATA_S_PENDING1) { |
| 2918 | DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " |
| 2919 | "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); |
| 2920 | TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = ( ccb)->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = ( ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1); } while (0); |
| 2921 | ccb_was_started = 0; |
| 2922 | } else if (ccb->ccb_xa.state == ATA_S_ONCHIP5 && ahci_port_intr(ap, |
| 2923 | 1 << ccb->ccb_slot)) { |
| 2924 | DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " |
| 2925 | "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); |
| 2926 | goto ret; |
| 2927 | } else if (ccb->ccb_xa.state != ATA_S_ONCHIP5) { |
| 2928 | DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " |
| 2929 | "handled%s\n", PORTNAME(ap), ccb->ccb_slot, |
| 2930 | ISSET(*active, 1 << ccb->ccb_slot) ? |
| 2931 | " but slot is still active?" : "."); |
| 2932 | goto ret; |
| 2933 | } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb ->ccb_slot)) |
| 2934 | AHCI_PREG_CI), 1 << ccb->ccb_slot)((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb ->ccb_slot)) && ISSET(*active,((*active) & (1 << ccb->ccb_slot)) |
| 2935 | 1 << ccb->ccb_slot)((*active) & (1 << ccb->ccb_slot))) { |
| 2936 | DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " |
| 2937 | "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), |
| 2938 | ccb->ccb_slot); |
| 2939 | *active &= ~(1 << ccb->ccb_slot); |
| 2940 | ccb->ccb_done(ccb); |
| 2941 | goto ret; |
| 2942 | } else { |
| 2943 | ccb_was_started = 1; |
| 2944 | } |
| 2945 | |
| 2946 | /* Complete the slot with a timeout error. */ |
| 2947 | ccb->ccb_xa.state = ATA_S_TIMEOUT4; |
| 2948 | *active &= ~(1 << ccb->ccb_slot); |
| 2949 | DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); |
| 2950 | ccb->ccb_done(ccb); /* This won't issue pending commands or run the |
| 2951 | atascsi completion. */ |
| 2952 | |
| 2953 | /* Reset port to abort running command. */ |
| 2954 | if (ccb_was_started) { |
| 2955 | DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " |
| 2956 | "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap), |
| 2957 | ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active); |
| 2958 | if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0) |
| 2959 | != 0) { |
| 2960 | printf("%s: failed to reset port during timeout " |
| 2961 | "handling, disabling it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 2962 | ap->ap_state = AP_S_FATAL_ERROR4; |
| 2963 | } |
| 2964 | |
| 2965 | /* Restart any other commands that were aborted by the reset. */ |
| 2966 | if (*active) { |
| 2967 | DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " |
| 2968 | "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", |
| 2969 | *active); |
| 2970 | if (ncq_cmd) |
| 2971 | ahci_pwrite(ap, AHCI_PREG_SACT0x34, *active); |
| 2972 | ahci_pwrite(ap, AHCI_PREG_CI0x38, *active); |
| 2973 | } |
| 2974 | } |
| 2975 | |
| 2976 | /* Issue any pending commands now. */ |
| 2977 | DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); |
| 2978 | if (ccb_was_started) |
| 2979 | ahci_issue_pending_commands(ap, ncq_cmd); |
| 2980 | else if (ap->ap_active == 0) |
| 2981 | ahci_issue_pending_ncq_commands(ap); |
| 2982 | |
| 2983 | /* Complete the timed out ata_xfer I/O (may generate new I/O). */ |
| 2984 | DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); |
| 2985 | ata_complete(xa); |
| 2986 | |
| 2987 | DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); |
| 2988 | ret: |
| 2989 | splx(s)spllower(s); |
| 2990 | } |
| 2991 | |
| 2992 | void |
| 2993 | ahci_empty_done(struct ahci_ccb *ccb) |
| 2994 | { |
| 2995 | if (ccb->ccb_xa.state != ATA_S_ERROR3) |
| 2996 | ccb->ccb_xa.state = ATA_S_COMPLETE2; |
| 2997 | } |
| 2998 | |
| 2999 | int |
| 3000 | ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap) |
| 3001 | { |
| 3002 | struct ahci_ccb *ccb; |
| 3003 | struct ata_fis_h2d *fis; |
| 3004 | int error; |
| 3005 | |
| 3006 | ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ |
| 3007 | ccb->ccb_xa.flags = ATA_F_POLL(1<<3) | ATA_F_GET_RFIS(1<<8); |
| 3008 | ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f; |
| 3009 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 3010 | |
| 3011 | memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 3012 | fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; |
| 3013 | fis->type = ATA_FIS_TYPE_H2D0x27; |
| 3014 | fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | SATA_PMP_CONTROL_PORT0x0f; |
| 3015 | fis->command = ATA_C_READ_PM0xe4; |
| 3016 | fis->features = which; |
| 3017 | fis->device = target | ATA_H2D_DEVICE_LBA0x40; |
| 3018 | fis->control = ATA_FIS_CONTROL_4BIT0x08; |
| 3019 | |
| 3020 | if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) { |
| 3021 | error = 1; |
| 3022 | } else { |
| 3023 | *datap = ccb->ccb_xa.rfis.sector_count | |
| 3024 | (ccb->ccb_xa.rfis.lba_low << 8) | |
| 3025 | (ccb->ccb_xa.rfis.lba_mid << 16) | |
| 3026 | (ccb->ccb_xa.rfis.lba_high << 24); |
| 3027 | error = 0; |
| 3028 | } |
| 3029 | ahci_put_pmp_ccb(ccb); |
| 3030 | return (error); |
| 3031 | } |
| 3032 | |
| 3033 | int |
| 3034 | ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data) |
| 3035 | { |
| 3036 | struct ahci_ccb *ccb; |
| 3037 | struct ata_fis_h2d *fis; |
| 3038 | int error; |
| 3039 | |
| 3040 | ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */ |
| 3041 | ccb->ccb_xa.flags = ATA_F_POLL(1<<3); |
| 3042 | ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f; |
| 3043 | ccb->ccb_xa.state = ATA_S_PENDING1; |
| 3044 | |
| 3045 | memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table))); |
| 3046 | fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; |
| 3047 | fis->type = ATA_FIS_TYPE_H2D0x27; |
| 3048 | fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | SATA_PMP_CONTROL_PORT0x0f; |
| 3049 | fis->command = ATA_C_WRITE_PM0xe8; |
| 3050 | fis->features = which; |
| 3051 | fis->device = target | ATA_H2D_DEVICE_LBA0x40; |
| 3052 | fis->sector_count = (u_int8_t)data; |
| 3053 | fis->lba_low = (u_int8_t)(data >> 8); |
| 3054 | fis->lba_mid = (u_int8_t)(data >> 16); |
| 3055 | fis->lba_high = (u_int8_t)(data >> 24); |
| 3056 | fis->control = ATA_FIS_CONTROL_4BIT0x08; |
| 3057 | |
| 3058 | error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout); |
| 3059 | ahci_put_pmp_ccb(ccb); |
| 3060 | return (error); |
| 3061 | } |
| 3062 | |
| 3063 | int |
| 3064 | ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap) |
| 3065 | { |
| 3066 | int error; |
| 3067 | |
| 3068 | error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS0, datap); |
| 3069 | if (error == 0) |
| 3070 | error = ahci_pmp_write(ap, target, SATA_PMREG_SERR1, -1); |
| 3071 | if (error) |
| 3072 | *datap = 0; |
| 3073 | |
| 3074 | return (error); |
| 3075 | } |
| 3076 | |
| 3077 | int |
| 3078 | ahci_pmp_identify(struct ahci_port *ap, int *ret_nports) |
| 3079 | { |
| 3080 | u_int32_t chipid; |
| 3081 | u_int32_t rev; |
| 3082 | u_int32_t nports; |
| 3083 | u_int32_t features; |
| 3084 | u_int32_t enabled; |
| 3085 | int s; |
| 3086 | |
| 3087 | s = splbio()splraise(0x3); |
| 3088 | |
| 3089 | if (ahci_pmp_read(ap, 15, 0, &chipid) || |
| 3090 | ahci_pmp_read(ap, 15, 1, &rev) || |
| 3091 | ahci_pmp_read(ap, 15, 2, &nports) || |
| 3092 | ahci_pmp_read(ap, 15, SATA_PMREG_FEA64, &features) || |
| 3093 | ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN96, &enabled)) { |
| 3094 | printf("%s: port multiplier identification failed\n", |
| 3095 | PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname)); |
| 3096 | splx(s)spllower(s); |
| 3097 | return (1); |
| 3098 | } |
| 3099 | splx(s)spllower(s); |
| 3100 | |
| 3101 | nports &= 0x0F; |
| 3102 | |
| 3103 | /* ignore SEMB port on SiI3726 port multiplier chips */ |
| 3104 | if (chipid == 0x37261095) { |
| 3105 | nports--; |
| 3106 | } |
| 3107 | |
| 3108 | printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, " |
| 3109 | "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), chipid, rev, |
| 3110 | SATA_PFMT_PM_REV"\20" "\003PM1.1" "\002PM1.0", nports, features, SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST", enabled, |
| 3111 | SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST"); |
| 3112 | |
| 3113 | *ret_nports = nports; |
| 3114 | return (0); |
| 3115 | } |
| 3116 | |
| 3117 | |
| 3118 | #ifdef HIBERNATE1 |
| 3119 | void |
| 3120 | ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb) |
| 3121 | { |
| 3122 | ccb->ccb_cmd_hdr->prdbc = 0; |
| 3123 | ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot); |
| 3124 | } |
| 3125 | |
| 3126 | int |
| 3127 | ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb) |
| 3128 | { |
| 3129 | u_int32_t is, ci_saved; |
| 3130 | int process_error = 0; |
| 3131 | |
| 3132 | is = ahci_pread(ap, AHCI_PREG_IS0x10); |
| 3133 | |
| 3134 | ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38); |
| 3135 | |
| 3136 | if (is & AHCI_PREG_IS_DHRS(1<<0)) { |
| 3137 | u_int32_t tfd; |
| 3138 | u_int32_t cmd; |
| 3139 | |
| 3140 | tfd = ahci_pread(ap, AHCI_PREG_TFD0x20); |
| 3141 | cmd = ahci_pread(ap, AHCI_PREG_CMD0x18); |
| 3142 | if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) && |
| 3143 | (cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) { |
| 3144 | process_error = 1; |
| 3145 | } else { |
| 3146 | ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0)); |
| 3147 | } |
| 3148 | } else if (is & (AHCI_PREG_IS_TFES(1<<30) | AHCI_PREG_IS_HBFS(1<<29) | |
| 3149 | AHCI_PREG_IS_IFS(1<<27) | AHCI_PREG_IS_OFS(1<<24) | AHCI_PREG_IS_UFS(1<<4))) { |
| 3150 | process_error = 1; |
| 3151 | } |
| 3152 | |
| 3153 | /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ |
| 3154 | if (process_error) { |
| 3155 | |
| 3156 | /* Turn off ST to clear CI and SACT. */ |
| 3157 | ahci_port_stop(ap, 0); |
| 3158 | |
| 3159 | /* just return an error indicator? we can't meaningfully |
| 3160 | * recover, and on the way back out we'll DVACT_RESUME which |
| 3161 | * resets and reinits the port. |
| 3162 | */ |
| 3163 | return (EIO5); |
| 3164 | } |
| 3165 | |
| 3166 | /* command is finished when the bit in CI for the slot goes to 0 */ |
| 3167 | if (ci_saved & (1 << ccb->ccb_slot)) { |
| 3168 | return (EAGAIN35); |
| 3169 | } |
| 3170 | |
| 3171 | return (0); |
| 3172 | } |
| 3173 | |
| 3174 | void |
| 3175 | ahci_hibernate_load_prdt(struct ahci_ccb *ccb) |
| 3176 | { |
| 3177 | struct ata_xfer *xa = &ccb->ccb_xa; |
| 3178 | struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; |
| 3179 | struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; |
| 3180 | int i; |
| 3181 | paddr_t data_phys; |
| 3182 | u_int64_t data_bus_phys; |
| 3183 | vaddr_t data_addr; |
| 3184 | size_t seglen; |
| 3185 | size_t buflen; |
| 3186 | |
| 3187 | if (xa->datalen == 0) { |
| 3188 | ccb->ccb_cmd_hdr->prdtl = 0; |
| 3189 | return; |
| 3190 | } |
| 3191 | |
| 3192 | /* derived from i386/amd64 _bus_dma_load_buffer; |
| 3193 | * for amd64 the buffer will always be dma safe. |
| 3194 | */ |
| 3195 | |
| 3196 | buflen = xa->datalen; |
| 3197 | data_addr = (vaddr_t)xa->data; |
| 3198 | for (i = 0; buflen > 0; i++) { |
| 3199 | pmap_extract(pmap_kernel()(&kernel_pmap_store), data_addr, &data_phys); |
| 3200 | data_bus_phys = data_phys; |
| 3201 | |
| 3202 | seglen = PAGE_SIZE(1 << 12) - ((u_long)data_addr & PGOFSET((1 << 12) - 1)); |
| 3203 | if (buflen < seglen) |
| 3204 | seglen = buflen; |
| 3205 | |
| 3206 | ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0); |
| 3207 | |
| 3208 | data_addr += seglen; |
| 3209 | buflen -= seglen; |
| 3210 | } |
| 3211 | |
| 3212 | htolem16(&cmd_slot->prdtl, i)(*(__uint16_t *)(&cmd_slot->prdtl) = ((__uint16_t)(i)) ); |
| 3213 | } |
| 3214 | |
| 3215 | int |
| 3216 | ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, |
| 3217 | int op, void *page) |
| 3218 | { |
| 3219 | /* we use the 'real' ahci_port and ahci_softc here, but |
| 3220 | * never write to them |
| 3221 | */ |
| 3222 | struct { |
| 3223 | struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */ |
| 3224 | struct ahci_rfis rfis; /* 1k aligned, 256 bytes */ |
| 3225 | /* cmd table isn't actually used because of mysteries */ |
| 3226 | struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */ |
| 3227 | struct ahci_port *ap; |
| 3228 | struct ahci_ccb ccb_buf; |
| 3229 | struct ahci_ccb *ccb; |
| 3230 | struct ahci_cmd_hdr *hdr_buf; |
| 3231 | int pmp_port; |
| 3232 | daddr_t poffset; |
| 3233 | size_t psize; |
| 3234 | } *my = page; |
| 3235 | struct ata_fis_h2d *fis; |
| 3236 | u_int32_t sector_count; |
| 3237 | struct ahci_cmd_hdr *cmd_slot; |
| 3238 | int rc; |
| 3239 | int timeout; |
| 3240 | u_int16_t flags; |
| 3241 | |
| 3242 | if (op == HIB_INIT-1) { |
| 3243 | struct device *disk; |
| 3244 | struct device *scsibus; |
| 3245 | struct ahci_softc *sc; |
| 3246 | extern struct cfdriver sd_cd; |
| 3247 | struct scsi_link *link; |
| 3248 | struct scsibus_softc *bus_sc; |
| 3249 | int port; |
| 3250 | paddr_t page_phys; |
| 3251 | u_int64_t item_phys; |
| 3252 | u_int32_t cmd; |
| 3253 | |
| 3254 | my->poffset = blkno; |
| 3255 | my->psize = size; |
| 3256 | |
| 3257 | /* map dev to an ahci port */ |
| 3258 | disk = disk_lookup(&sd_cd, DISKUNIT(dev)(((unsigned)((dev) & 0xff) | (((dev) & 0xffff0000) >> 8)) / 16)); |
| 3259 | scsibus = disk->dv_parent; |
| 3260 | sc = (struct ahci_softc *)disk->dv_parent->dv_parent; |
| 3261 | |
| 3262 | /* find the scsi_link for the device, which has the port */ |
| 3263 | port = -1; |
| 3264 | bus_sc = (struct scsibus_softc *)scsibus; |
| 3265 | SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list)for((link) = ((&bus_sc->sc_link_list)->slh_first); ( link) != ((void *)0); (link) = ((link)->bus_list.sle_next) ) { |
| 3266 | if (link->device_softc == disk) { |
| 3267 | port = link->target; |
| 3268 | if (link->lun > 0) |
| 3269 | my->pmp_port = link->lun - 1; |
| 3270 | else |
| 3271 | my->pmp_port = 0; |
| 3272 | |
| 3273 | break; |
| 3274 | } |
| 3275 | } |
| 3276 | if (port == -1) { |
| 3277 | /* don't know where the disk is */ |
| 3278 | return (EIO5); |
| 3279 | } |
| 3280 | |
| 3281 | my->ap = sc->sc_ports[port]; |
| 3282 | |
| 3283 | /* we're going to use the first command slot, |
| 3284 | * so ensure it's not already in use |
| 3285 | */ |
| 3286 | if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT6) { |
| 3287 | /* this shouldn't happen, we should be idle */ |
| 3288 | return (EIO5); |
| 3289 | } |
| 3290 | |
| 3291 | /* stop the port so we can relocate to the hibernate page */ |
| 3292 | if (ahci_port_stop(my->ap, 1)) { |
| 3293 | return (EIO5); |
| 3294 | } |
| 3295 | ahci_pwrite(my->ap, AHCI_PREG_SCTL0x2c, 0); |
| 3296 | |
| 3297 | pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)page, &page_phys); |
| 3298 | |
| 3299 | /* Setup RFIS base address */ |
| 3300 | item_phys = page_phys + ((void *)&my->rfis - page); |
| 3301 | ahci_pwrite(my->ap, AHCI_PREG_FBU0x0c, |
| 3302 | (u_int32_t)(item_phys >> 32)); |
| 3303 | ahci_pwrite(my->ap, AHCI_PREG_FB0x08, (u_int32_t)item_phys); |
| 3304 | |
| 3305 | /* Enable FIS reception and activate port. */ |
| 3306 | cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 3307 | cmd |= AHCI_PREG_CMD_FRE(1<<4) | AHCI_PREG_CMD_POD(1<<2) | |
| 3308 | AHCI_PREG_CMD_SUD(1<<1); |
| 3309 | ahci_pwrite(my->ap, AHCI_PREG_CMD0x18, cmd | |
| 3310 | AHCI_PREG_CMD_ICC_ACTIVE0x10000000); |
| 3311 | |
| 3312 | /* Check whether port activated. */ |
| 3313 | cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000; |
| 3314 | if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) { |
| 3315 | return (EIO5); |
| 3316 | } |
| 3317 | |
| 3318 | /* Set up the single CCB */ |
| 3319 | my->ccb = &my->ccb_buf; |
| 3320 | my->ccb->ccb_slot = 0; |
| 3321 | my->ccb->ccb_port = my->ap; |
| 3322 | |
| 3323 | /* Setup command list base address */ |
| 3324 | item_phys = page_phys + ((void *)&my->cmd_hdr - page); |
| 3325 | ahci_pwrite(my->ap, AHCI_PREG_CLBU0x04, |
| 3326 | (u_int32_t)(item_phys >> 32)); |
| 3327 | ahci_pwrite(my->ap, AHCI_PREG_CLB0x00, (u_int32_t)item_phys); |
| 3328 | |
| 3329 | my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0]; |
| 3330 | |
| 3331 | /* use existing cmd table - moving to a new one fails */ |
| 3332 | my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table; |
| 3333 | pmap_extract(pmap_kernel()(&kernel_pmap_store), |
| 3334 | (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table)((void *)(my->ap->ap_dmamem_cmd_table)->adm_kva), |
| 3335 | &page_phys); |
| 3336 | item_phys = page_phys; |
| 3337 | #if 0 |
| 3338 | /* use cmd table in hibernate page (doesn't work) */ |
| 3339 | my->ccb->ccb_cmd_table = &my->cmd_table; |
| 3340 | item_phys = page_phys + ((void *)&my->cmd_table - page); |
| 3341 | #endif |
| 3342 | htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys)(*(__uint64_t *)(&my->ccb->ccb_cmd_hdr->ctba) = ( (__uint64_t)(item_phys))); |
| 3343 | |
| 3344 | my->ccb->ccb_xa.fis = |
| 3345 | (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis; |
| 3346 | my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd; |
| 3347 | my->ccb->ccb_xa.tag = 0; |
| 3348 | |
| 3349 | /* Wait for ICC change to complete */ |
| 3350 | ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((my->ap), (0x18), (0xf0000000), 0, (1)); |
| 3351 | |
| 3352 | if (ahci_port_start(my->ap, 0)((my->ap)->ap_sc->sc_port_start((my->ap), (0)))) { |
| 3353 | return (EIO5); |
| 3354 | } |
| 3355 | |
| 3356 | /* Flush interrupts for port */ |
| 3357 | ahci_pwrite(my->ap, AHCI_PREG_IS0x10, ahci_pread(my->ap, |
| 3358 | AHCI_PREG_IS0x10)); |
| 3359 | ahci_write(sc, AHCI_REG_IS0x008, 1 << port); |
| 3360 | |
| 3361 | ahci_enable_interrupts(my->ap); |
| 3362 | return (0); |
| 3363 | } else if (op == HIB_DONE-2) { |
| 3364 | ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME4); |
| 3365 | return (0); |
| 3366 | } |
| 3367 | |
| 3368 | if (blkno > my->psize) |
| 3369 | return (E2BIG7); |
| 3370 | blkno += my->poffset; |
| 3371 | |
| 3372 | /* build fis */ |
| 3373 | sector_count = size / 512; /* dlg promises this is okay */ |
| 3374 | my->ccb->ccb_xa.flags = op == HIB_W1 ? ATA_F_WRITE(1<<1) : ATA_F_READ(1<<0); |
| 3375 | fis = my->ccb->ccb_xa.fis; |
| 3376 | fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | my->pmp_port; |
| 3377 | fis->lba_low = blkno & 0xff; |
| 3378 | fis->lba_mid = (blkno >> 8) & 0xff; |
| 3379 | fis->lba_high = (blkno >> 16) & 0xff; |
| 3380 | |
| 3381 | if (sector_count > 0x100 || blkno > 0xfffffff) { |
| 3382 | /* Use LBA48 */ |
| 3383 | fis->command = op == HIB_W1 ? ATA_C_WRITEDMA_EXT0x35 : |
| 3384 | ATA_C_READDMA_EXT0x25; |
| 3385 | fis->device = ATA_H2D_DEVICE_LBA0x40; |
| 3386 | fis->lba_low_exp = (blkno >> 24) & 0xff; |
| 3387 | fis->lba_mid_exp = (blkno >> 32) & 0xff; |
| 3388 | fis->lba_high_exp = (blkno >> 40) & 0xff; |
| 3389 | fis->sector_count = sector_count & 0xff; |
| 3390 | fis->sector_count_exp = (sector_count >> 8) & 0xff; |
| 3391 | } else { |
| 3392 | /* Use LBA */ |
| 3393 | fis->command = op == HIB_W1 ? ATA_C_WRITEDMA0xca : ATA_C_READDMA0xc8; |
| 3394 | fis->device = ATA_H2D_DEVICE_LBA0x40 | ((blkno >> 24) & 0x0f); |
| 3395 | fis->sector_count = sector_count & 0xff; |
| 3396 | } |
| 3397 | |
| 3398 | my->ccb->ccb_xa.data = (void *)addr; |
| 3399 | my->ccb->ccb_xa.datalen = size; |
| 3400 | my->ccb->ccb_xa.pmp_port = my->pmp_port; |
| 3401 | my->ccb->ccb_xa.flags |= ATA_F_POLL(1<<3); |
| 3402 | |
| 3403 | cmd_slot = my->ccb->ccb_cmd_hdr; |
| 3404 | flags = 5; /* FIS length (in DWORDs) */ |
| 3405 | flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12; |
| 3406 | |
| 3407 | if (op == HIB_W1) |
| 3408 | flags |= AHCI_CMD_LIST_FLAG_W(1<<6); |
| 3409 | |
| 3410 | htolem16(&cmd_slot->flags, flags)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(flags ))); |
| 3411 | |
| 3412 | ahci_hibernate_load_prdt(my->ccb); |
| 3413 | |
| 3414 | ahci_hibernate_io_start(my->ap, my->ccb); |
| 3415 | timeout = 1000000; |
| 3416 | while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN35) { |
Although the value stored to 'rc' is used in the enclosing expression, the value is never actually read from 'rc' | |
| 3417 | delay(1)(*delay_func)(1); |
| 3418 | timeout--; |
| 3419 | if (timeout == 0) { |
| 3420 | return (EIO5); |
| 3421 | } |
| 3422 | } |
| 3423 | |
| 3424 | return (0); |
| 3425 | } |
| 3426 | |
| 3427 | #endif |