Bug Summary

File:dev/ic/ahci.c
Warning:line 2081, column 3
Value stored to 'serr' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ahci.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/ic/ahci.c
1/* $OpenBSD: ahci.c,v 1.39 2023/02/03 18:31:16 miod Exp $ */
2
3/*
4 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
5 * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com>
6 * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org>
7 *
8 * Permission to use, copy, modify, and distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 */
20
21#include <sys/param.h>
22#include <sys/systm.h>
23#include <sys/buf.h>
24#include <sys/kernel.h>
25#include <sys/malloc.h>
26#include <sys/device.h>
27#include <sys/queue.h>
28#include <sys/mutex.h>
29#include <sys/pool.h>
30
31#include <machine/bus.h>
32
33#include <dev/ic/ahcireg.h>
34#include <dev/ic/ahcivar.h>
35
36#ifdef AHCI_DEBUG
37#define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
38 while (0)
39#define AHCI_D_TIMEOUT 0x00
40#define AHCI_D_VERBOSE 0x01
41#define AHCI_D_INTR 0x02
42#define AHCI_D_XFER 0x08
43int ahcidebug = AHCI_D_VERBOSE;
44#else
45#define DPRINTF(m, f...)
46#endif
47
48#ifdef HIBERNATE1
49#include <uvm/uvm_extern.h>
50#include <sys/hibernate.h>
51#include <sys/disk.h>
52#include <sys/disklabel.h>
53
54#include <scsi/scsi_all.h>
55#include <scsi/scsiconf.h>
56
57void ahci_hibernate_io_start(struct ahci_port *,
58 struct ahci_ccb *);
59int ahci_hibernate_io_poll(struct ahci_port *,
60 struct ahci_ccb *);
61void ahci_hibernate_load_prdt(struct ahci_ccb *);
62
63int ahci_hibernate_io(dev_t dev, daddr_t blkno,
64 vaddr_t addr, size_t size, int wr, void *page);
65#endif
66
67struct cfdriver ahci_cd = {
68 NULL((void *)0), "ahci", DV_DULL
69};
70
71void ahci_enable_interrupts(struct ahci_port *);
72
73int ahci_init(struct ahci_softc *);
74int ahci_port_alloc(struct ahci_softc *, u_int);
75void ahci_port_detect(struct ahci_softc *, u_int);
76void ahci_port_free(struct ahci_softc *, u_int);
77int ahci_port_init(struct ahci_softc *, u_int);
78
79int ahci_default_port_start(struct ahci_port *, int);
80int ahci_port_stop(struct ahci_port *, int);
81int ahci_port_clo(struct ahci_port *);
82int ahci_port_softreset(struct ahci_port *);
83void ahci_port_comreset(struct ahci_port *);
84int ahci_port_portreset(struct ahci_port *, int);
85void ahci_port_portreset_start(struct ahci_port *);
86int ahci_port_portreset_poll(struct ahci_port *);
87void ahci_port_portreset_wait(struct ahci_port *);
88int ahci_port_portreset_finish(struct ahci_port *, int);
89int ahci_port_signature(struct ahci_port *);
90int ahci_pmp_port_softreset(struct ahci_port *, int);
91int ahci_pmp_port_portreset(struct ahci_port *, int);
92int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
93
94int ahci_load_prdt(struct ahci_ccb *);
95void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
96 u_int32_t, u_int32_t);
97void ahci_unload_prdt(struct ahci_ccb *);
98
99int ahci_poll(struct ahci_ccb *, int, void (*)(void *));
100void ahci_start(struct ahci_ccb *);
101
102void ahci_issue_pending_ncq_commands(struct ahci_port *);
103void ahci_issue_pending_commands(struct ahci_port *, int);
104
105int ahci_intr(void *);
106u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
107
108struct ahci_ccb *ahci_get_ccb(struct ahci_port *);
109void ahci_put_ccb(struct ahci_ccb *);
110
111struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
112void ahci_put_err_ccb(struct ahci_ccb *);
113
114struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *);
115void ahci_put_pmp_ccb(struct ahci_ccb *);
116
117int ahci_port_read_ncq_error(struct ahci_port *, int *, int);
118
119struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t);
120void ahci_dmamem_free(struct ahci_softc *,
121 struct ahci_dmamem *);
122
123u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
124void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
125int ahci_wait_ne(struct ahci_softc *, bus_size_t,
126 u_int32_t, u_int32_t);
127
128u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
129void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
130int ahci_pwait_eq(struct ahci_port *, bus_size_t,
131 u_int32_t, u_int32_t, int);
132void ahci_flush_tfd(struct ahci_port *ap);
133u_int32_t ahci_active_mask(struct ahci_port *);
134int ahci_port_detect_pmp(struct ahci_port *);
135void ahci_pmp_probe_timeout(void *);
136
137/* pmp operations */
138int ahci_pmp_read(struct ahci_port *, int, int,
139 u_int32_t *);
140int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
141int ahci_pmp_phy_status(struct ahci_port *, int,
142 u_int32_t *);
143int ahci_pmp_identify(struct ahci_port *, int *);
144
145
146/* Wait for all bits in _b to be cleared */
147#define ahci_pwait_clr(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) \
148 ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
149
150/* Wait for all bits in _b to be set */
151#define ahci_pwait_set(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) \
152 ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
153
154
155
156/* provide methods for atascsi to call */
157int ahci_ata_probe(void *, int, int);
158void ahci_ata_free(void *, int, int);
159struct ata_xfer * ahci_ata_get_xfer(void *, int);
160void ahci_ata_put_xfer(struct ata_xfer *);
161void ahci_ata_cmd(struct ata_xfer *);
162
163const struct atascsi_methods ahci_atascsi_methods = {
164 ahci_ata_probe,
165 ahci_ata_free,
166 ahci_ata_get_xfer,
167 ahci_ata_put_xfer,
168 ahci_ata_cmd
169};
170
171/* ccb completions */
172void ahci_ata_cmd_done(struct ahci_ccb *);
173void ahci_pmp_cmd_done(struct ahci_ccb *);
174void ahci_ata_cmd_timeout(void *);
175void ahci_empty_done(struct ahci_ccb *);
176
177int
178ahci_attach(struct ahci_softc *sc)
179{
180 struct atascsi_attach_args aaa;
181 u_int32_t pi;
182 int i, j, done;
183
184 if (sc->sc_port_start == NULL((void *)0))
185 sc->sc_port_start = ahci_default_port_start;
186
187 if (ahci_init(sc) != 0) {
188 /* error already printed by ahci_init */
189 goto unmap;
190 }
191
192 printf("\n");
193
194 sc->sc_cap = ahci_read(sc, AHCI_REG_CAP0x000);
195 sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap)((((sc->sc_cap) & 0x1f00)>>8)+1);
196#ifdef AHCI_DEBUG
197 if (ahcidebug & AHCI_D_VERBOSE) {
198 const char *gen;
199
200 switch (sc->sc_cap & AHCI_REG_CAP_ISS(0xf<<20)) {
201 case AHCI_REG_CAP_ISS_G1(0x1<<20):
202 gen = "1 (1.5Gbps)";
203 break;
204 case AHCI_REG_CAP_ISS_G2(0x2<<20):
205 gen = "2 (3.0Gb/s)";
206 break;
207 case AHCI_REG_CAP_ISS_G3(0x3<<20):
208 gen = "3 (6.0Gb/s)";
209 break;
210 default:
211 gen = "unknown";
212 break;
213 }
214
215 printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
216 DEVNAME(sc)((sc)->sc_dev.dv_xname), sc->sc_cap, AHCI_FMT_CAP"\020" "\040S64A" "\037NCQ" "\036SSNTF" "\035SMPS" "\034SSS" "\033SALP"
"\032SAL" "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" "\021FBSS"
"\020PMD" "\017SSC" "\016PSC" "\010CCCS" "\007EMS" "\006SXS"
,
217 AHCI_REG_CAP_NP(sc->sc_cap)(((sc->sc_cap) & 0x1f)+1), sc->sc_ncmds, gen);
218 printf("%s: extended capabilities 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
219 ahci_read(sc, AHCI_REG_CAP20x024), AHCI_FMT_CAP2"\020" "\006DESO" "\005SADM" "\004SDS" "\003APST" "\002NVMP" "\001BOH");
220 }
221#endif
222
223 pi = ahci_read(sc, AHCI_REG_PI0x00c);
224 DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
225 DEVNAME(sc), pi);
226
227#ifdef AHCI_COALESCE
228 /* Naive coalescing support - enable for all ports. */
229 if (sc->sc_cap & AHCI_REG_CAP_CCCS(1<<7)) {
230 u_int16_t ccc_timeout = 20;
231 u_int8_t ccc_numcomplete = 12;
232 u_int32_t ccc_ctl;
233
234 /* disable coalescing during reconfiguration. */
235 ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL0x014);
236 ccc_ctl &= ~0x00000001;
237 ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl);
238
239 sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl)(((ccc_ctl) & 0xf8) >> 3);
240 if (pi & sc->sc_ccc_mask) {
241 /* A conflict with the implemented port list? */
242 printf("%s: coalescing interrupt/implemented port list "
243 "conflict, PI: %08x, ccc_mask: %08x\n",
244 DEVNAME(sc)((sc)->sc_dev.dv_xname), pi, sc->sc_ccc_mask);
245 sc->sc_ccc_mask = 0;
246 goto noccc;
247 }
248
249 /* ahci_port_start will enable each port when it starts. */
250 sc->sc_ccc_ports = pi;
251 sc->sc_ccc_ports_cur = 0;
252
253 /* program thresholds and enable overall coalescing. */
254 ccc_ctl &= ~0xffffff00;
255 ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
256 ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl);
257 ahci_write(sc, AHCI_REG_CCC_PORTS0x018, 0);
258 ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl | 1);
259 }
260noccc:
261#endif
262 /*
263 * Given that ahci_port_alloc() will grab one CCB for error recovery
264 * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds
265 * pretend at least 2 command slots for devices without NCQ support.
266 * That way, also at least 1 slot is made available for atascsi(4).
267 */
268 sc->sc_ncmds = max(2, sc->sc_ncmds);
269 for (i = 0; i < AHCI_MAX_PORTS32; i++) {
270 if (!ISSET(pi, 1 << i)((pi) & (1 << i))) {
271 /* dont allocate stuff if the port isnt implemented */
272 continue;
273 }
274
275 if (ahci_port_alloc(sc, i) == ENOMEM12)
276 goto freeports;
277
278 if (sc->sc_ports[i] != NULL((void *)0))
279 ahci_port_portreset_start(sc->sc_ports[i]);
280 }
281
282 /*
283 * Poll for device detection until all ports report a device, or one
284 * second has elapsed.
285 */
286 for (i = 0; i < 1000; i++) {
287 done = 1;
288 for (j = 0; j < AHCI_MAX_PORTS32; j++) {
289 if (sc->sc_ports[j] == NULL((void *)0))
290 continue;
291
292 if (ahci_port_portreset_poll(sc->sc_ports[j]))
293 done = 0;
294 }
295
296 if (done)
297 break;
298
299 delay(1000)(*delay_func)(1000);
300 }
301
302 /*
303 * Finish device detection on all ports that initialized.
304 */
305 for (i = 0; i < AHCI_MAX_PORTS32; i++) {
306 if (sc->sc_ports[i] != NULL((void *)0))
307 ahci_port_detect(sc, i);
308 }
309
310 memset(&aaa, 0, sizeof(aaa))__builtin_memset((&aaa), (0), (sizeof(aaa)));
311 aaa.aaa_cookie = sc;
312 aaa.aaa_methods = &ahci_atascsi_methods;
313 aaa.aaa_minphys = NULL((void *)0);
314 aaa.aaa_nports = AHCI_MAX_PORTS32;
315 aaa.aaa_ncmds = sc->sc_ncmds - 1;
316 if (!(sc->sc_flags & AHCI_F_NO_NCQ(1<<0)) &&
317 sc->sc_ncmds > 2 &&
318 (sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30))) {
319 aaa.aaa_capability |= ASAA_CAP_NCQ(1 << 0) | ASAA_CAP_PMP_NCQ(1 << 2);
320 }
321
322 sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
323
324 /* Enable interrupts */
325 ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) | AHCI_REG_GHC_IE(1<<1));
326
327 return 0;
328
329freeports:
330 for (i = 0; i < AHCI_MAX_PORTS32; i++) {
331 if (sc->sc_ports[i] != NULL((void *)0))
332 ahci_port_free(sc, i);
333 }
334unmap:
335 /* Disable controller */
336 ahci_write(sc, AHCI_REG_GHC0x004, 0);
337 return 1;
338}
339
340int
341ahci_detach(struct ahci_softc *sc, int flags)
342{
343 int rv, i;
344
345 if (sc->sc_atascsi != NULL((void *)0)) {
346 rv = atascsi_detach(sc->sc_atascsi, flags);
347 if (rv != 0)
348 return (rv);
349 }
350
351 for (i = 0; i < AHCI_MAX_PORTS32; i++) {
352 if (sc->sc_ports[i] != NULL((void *)0))
353 ahci_port_free(sc, i);
354 }
355
356 return (0);
357}
358
359int
360ahci_activate(struct device *self, int act)
361{
362 struct ahci_softc *sc = (struct ahci_softc *)self;
363 int i, rv = 0;
364
365 switch (act) {
366 case DVACT_RESUME4:
367 /* enable ahci (global interrupts disabled) */
368 ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31));
369
370 /* restore BIOS initialised parameters */
371 ahci_write(sc, AHCI_REG_CAP0x000, sc->sc_cap);
372
373 for (i = 0; i < AHCI_MAX_PORTS32; i++) {
374 if (sc->sc_ports[i] != NULL((void *)0))
375 ahci_port_init(sc, i);
376 }
377
378 /* Enable interrupts */
379 ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) | AHCI_REG_GHC_IE(1<<1));
380
381 rv = config_activate_children(self, act);
382 break;
383 case DVACT_POWERDOWN6:
384 rv = config_activate_children(self, act);
385 for (i = 0; i < AHCI_MAX_PORTS32; i++) {
386 if (sc->sc_ports[i] != NULL((void *)0))
387 ahci_port_stop(sc->sc_ports[i], 1);
388 }
389 break;
390 default:
391 rv = config_activate_children(self, act);
392 break;
393 }
394 return (rv);
395}
396
397int
398ahci_init(struct ahci_softc *sc)
399{
400 u_int32_t reg, cap, pi;
401 const char *revision;
402
403 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
404 AHCI_FMT_GHC);
405
406 /* save BIOS initialised parameters, enable staggered spin up */
407 cap = ahci_read(sc, AHCI_REG_CAP0x000);
408 cap &= AHCI_REG_CAP_SMPS(1<<28);
409 cap |= AHCI_REG_CAP_SSS(1<<27);
410 pi = ahci_read(sc, AHCI_REG_PI0x00c);
411
412 if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))(((1<<31)) & (ahci_read(sc, 0x004)))) {
413 /* reset the controller */
414 ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0));
415 if (ahci_wait_ne(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0),
416 AHCI_REG_GHC_HR(1<<0)) != 0) {
417 printf(" unable to reset controller\n");
418 return (1);
419 }
420 }
421
422 /* enable ahci (global interrupts disabled) */
423 ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31));
424
425 /* restore parameters */
426 ahci_write(sc, AHCI_REG_CAP0x000, cap);
427 ahci_write(sc, AHCI_REG_PI0x00c, pi);
428
429 /* check the revision */
430 reg = ahci_read(sc, AHCI_REG_VS0x010);
431 switch (reg) {
432 case AHCI_REG_VS_0_950x00000905:
433 revision = "0.95";
434 break;
435 case AHCI_REG_VS_1_00x00010000:
436 revision = "1.0";
437 break;
438 case AHCI_REG_VS_1_10x00010100:
439 revision = "1.1";
440 break;
441 case AHCI_REG_VS_1_20x00010200:
442 revision = "1.2";
443 break;
444 case AHCI_REG_VS_1_30x00010300:
445 revision = "1.3";
446 break;
447 case AHCI_REG_VS_1_3_10x00010301:
448 revision = "1.3.1";
449 break;
450
451 default:
452 printf(" unsupported AHCI revision 0x%08x\n", reg);
453 return (1);
454 }
455
456 printf(" AHCI %s", revision);
457
458 return (0);
459}
460
461void
462ahci_enable_interrupts(struct ahci_port *ap)
463{
464 ahci_pwrite(ap, AHCI_PREG_IE0x14, AHCI_PREG_IE_TFEE(1<<30) | AHCI_PREG_IE_HBFE(1<<29) |
465 AHCI_PREG_IE_IFE(1<<27) | AHCI_PREG_IE_OFE(1<<24) | AHCI_PREG_IE_DPE(1<<5) |
466 AHCI_PREG_IE_UFE(1<<4) |
467 ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29)) ? AHCI_PREG_IE_IPME(1<<23) : 0) |
468#ifdef AHCI_COALESCE
469 ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
470 (AHCI_PREG_IE_SDBE(1<<3) | AHCI_PREG_IE_DHRE(1<<0)))
471#else
472 AHCI_PREG_IE_SDBE(1<<3) | AHCI_PREG_IE_DHRE(1<<0)
473#endif
474 );
475}
476
477int
478ahci_port_alloc(struct ahci_softc *sc, u_int port)
479{
480 struct ahci_port *ap;
481 struct ahci_ccb *ccb;
482 u_int64_t dva;
483 u_int32_t cmd;
484 struct ahci_cmd_hdr *hdr;
485 struct ahci_cmd_table *table;
486 int i, rc = ENOMEM12;
487
488 ap = malloc(sizeof(*ap), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008);
489 if (ap == NULL((void *)0)) {
490 printf("%s: unable to allocate memory for port %d\n",
491 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
492 goto reterr;
493 }
494 ap->ap_err_scratch = dma_alloc(DEV_BSIZE(1 << 9), PR_NOWAIT0x0002 | PR_ZERO0x0008);
495 if (ap->ap_err_scratch == NULL((void *)0)) {
496 printf("%s: unable to allocate DMA scratch buf for port %d\n",
497 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
498 free(ap, M_DEVBUF2, sizeof(*ap));
499 goto reterr;
500 }
501
502#ifdef AHCI_DEBUG
503 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
504 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
505#endif
506 ap->ap_port = port;
507 sc->sc_ports[port] = ap;
508
509 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
510 AHCI_PORT_REGION(port)(0x100 + ((port) * 0x80)), AHCI_PORT_SIZE0x80, &ap->ap_ioh) != 0) {
511 printf("%s: unable to create register window for port %d\n",
512 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
513 goto freeport;
514 }
515
516 ap->ap_sc = sc;
517#ifdef AHCI_COALESCE
518 ap->ap_num = port;
519#endif
520 TAILQ_INIT(&ap->ap_ccb_free)do { (&ap->ap_ccb_free)->tqh_first = ((void *)0); (
&ap->ap_ccb_free)->tqh_last = &(&ap->ap_ccb_free
)->tqh_first; } while (0)
;
521 TAILQ_INIT(&ap->ap_ccb_pending)do { (&ap->ap_ccb_pending)->tqh_first = ((void *)0)
; (&ap->ap_ccb_pending)->tqh_last = &(&ap->
ap_ccb_pending)->tqh_first; } while (0)
;
522 mtx_init(&ap->ap_ccb_mtx, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&ap->
ap_ccb_mtx), ((((0x3)) > 0x0 && ((0x3)) < 0x9) ?
0x9 : ((0x3)))); } while (0)
;
523
524 /* Disable port interrupts */
525 ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
526
527 /* Sec 10.1.2 - deinitialise port if it is already running */
528 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
529 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |((cmd) & (((1<<0) | (1<<15) | (1<<4) | (
1<<14))))
530 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR))((cmd) & (((1<<0) | (1<<15) | (1<<4) | (
1<<14))))
||
531 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) {
532 int r;
533
534 r = ahci_port_stop(ap, 1);
535 if (r) {
536 printf("%s: unable to disable %s, ignoring port %d\n",
537 DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port);
538 rc = ENXIO6;
539 goto freeport;
540 }
541
542 /* Write DET to zero */
543 ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
544 }
545
546 /* Allocate RFIS */
547 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
548 if (ap->ap_dmamem_rfis == NULL((void *)0))
549 goto nomem;
550
551 /* Setup RFIS base address */
552 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void *)(ap->ap_dmamem_rfis)->adm_kva);
553 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0
].ds_addr)
;
554 ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32));
555 ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva);
556
557 /* Enable FIS reception and activate port. */
558 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
559 cmd |= AHCI_PREG_CMD_FRE(1<<4) | AHCI_PREG_CMD_POD(1<<2) | AHCI_PREG_CMD_SUD(1<<1);
560 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd | AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
561
562 /* Check whether port activated. Skip it if not. */
563 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
564 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
565 rc = ENXIO6;
566 goto freeport;
567 }
568
569 /* Allocate a CCB for each command slot */
570 ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
571 M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008);
572 if (ap->ap_ccbs == NULL((void *)0)) {
573 printf("%s: unable to allocate command list for port %d\n",
574 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
575 goto freeport;
576 }
577
578 /* Command List Structures and Command Tables */
579 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
580 sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
581 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
582 sc->sc_ncmds * sizeof(struct ahci_cmd_table));
583 if (ap->ap_dmamem_cmd_table == NULL((void *)0) || ap->ap_dmamem_cmd_list == NULL((void *)0)) {
584nomem:
585 printf("%s: unable to allocate DMA memory for port %d\n",
586 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
587 goto freeport;
588 }
589
590 /* Setup command list base address */
591 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs
[0].ds_addr)
;
592 ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32));
593 ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva);
594
595 /* Split CCB allocation into CCBs and assign to command header/table */
596 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list)((void *)(ap->ap_dmamem_cmd_list)->adm_kva);
597 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table)((void *)(ap->ap_dmamem_cmd_table)->adm_kva);
598 for (i = 0; i < sc->sc_ncmds; i++) {
599 ccb = &ap->ap_ccbs[i];
600
601 if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64
* 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 | 0x0002),
(&ccb->ccb_dmamap))
602 (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64
* 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 | 0x0002),
(&ccb->ccb_dmamap))
603 &ccb->ccb_dmamap)(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64
* 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 | 0x0002),
(&ccb->ccb_dmamap))
!= 0) {
604 printf("%s: unable to create dmamap for port %d "
605 "ccb %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port, i);
606 goto freeport;
607 }
608
609 ccb->ccb_slot = i;
610 ccb->ccb_port = ap;
611 ccb->ccb_cmd_hdr = &hdr[i];
612 ccb->ccb_cmd_table = &table[i];
613 htolem64(&ccb->ccb_cmd_hdr->ctba,(*(__uint64_t *)(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t
)(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs
[0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table
))))
614 AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +(*(__uint64_t *)(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t
)(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs
[0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table
))))
615 ccb->ccb_slot * sizeof(struct ahci_cmd_table))(*(__uint64_t *)(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t
)(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs
[0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table
))))
;
616
617 ccb->ccb_xa.fis =
618 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
619 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
620 ccb->ccb_xa.tag = i;
621
622 ccb->ccb_xa.state = ATA_S_COMPLETE2;
623 ahci_put_ccb(ccb);
624 }
625
626 /* grab a ccb for use during error recovery */
627 ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
628 TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry)do { if (((ap->ap_ccb_err)->ccb_entry.tqe_next) != ((void
*)0)) (ap->ap_ccb_err)->ccb_entry.tqe_next->ccb_entry
.tqe_prev = (ap->ap_ccb_err)->ccb_entry.tqe_prev; else (
&ap->ap_ccb_free)->tqh_last = (ap->ap_ccb_err)->
ccb_entry.tqe_prev; *(ap->ap_ccb_err)->ccb_entry.tqe_prev
= (ap->ap_ccb_err)->ccb_entry.tqe_next; ((ap->ap_ccb_err
)->ccb_entry.tqe_prev) = ((void *)-1); ((ap->ap_ccb_err
)->ccb_entry.tqe_next) = ((void *)-1); } while (0)
;
629 ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE2;
630
631 /* Wait for ICC change to complete */
632 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1));
633 rc = 0;
634
635freeport:
636 if (rc != 0)
637 ahci_port_free(sc, port);
638reterr:
639 return (rc);
640}
641
642void
643ahci_port_detect(struct ahci_softc *sc, u_int port)
644{
645 struct ahci_port *ap;
646 const char *speed;
647 int rc;
648
649 ap = sc->sc_ports[port];
650
651 rc = ahci_port_portreset_finish(ap, 1);
652 switch (rc) {
653 case ENODEV19:
654 switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
655 case AHCI_PREG_SSTS_DET_DEV_NE0x1:
656 printf("%s: device not communicating on port %d\n",
657 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
658 break;
659 case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4:
660 printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
661 port);
662 break;
663 default:
664 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
665 "on port %d\n", DEVNAME(sc), port);
666 break;
667 }
668 goto freeport;
669
670 case EBUSY16:
671 printf("%s: device on port %d didn't come ready, "
672 "TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port,
673 ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
674
675 /* Try a soft reset to clear busy */
676 rc = ahci_port_softreset(ap);
677 if (rc) {
678 printf("%s: unable to communicate "
679 "with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
680 goto freeport;
681 }
682 break;
683
684 default:
685 break;
686 }
687
688 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
689 DEVNAME(sc), port, rc);
690
691 /* Read current link speed */
692 switch(ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_SPD0xf0) {
693 case AHCI_PREG_SSTS_SPD_GEN10x10:
694 speed = "1.5Gb/s";
695 break;
696 case AHCI_PREG_SSTS_SPD_GEN20x20:
697 speed = "3.0Gb/s";
698 break;
699 case AHCI_PREG_SSTS_SPD_GEN30x30:
700 speed = "6.0Gb/s";
701 break;
702 default:
703 speed = NULL((void *)0);
704 break;
705 }
706 if (speed != NULL((void *)0))
707 printf("%s: port %d: %s\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), port, speed);
708
709 /* Enable command transfers on port */
710 if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
711 printf("%s: failed to start command DMA on port %d, "
712 "disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
713 rc = ENXIO6; /* couldn't start port */
714 }
715
716 /* Flush interrupts for port */
717 ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
718 ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
719
720 ahci_enable_interrupts(ap);
721freeport:
722 if (rc != 0)
723 ahci_port_free(sc, port);
724}
725
726void
727ahci_port_free(struct ahci_softc *sc, u_int port)
728{
729 struct ahci_port *ap = sc->sc_ports[port];
730 struct ahci_ccb *ccb;
731
732 /* Ensure port is disabled and its interrupts are flushed */
733 if (ap->ap_sc) {
734 ahci_pwrite(ap, AHCI_PREG_CMD0x18, 0);
735 ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
736 ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
737 ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
738 }
739
740 if (ap->ap_ccb_err)
741 ahci_put_ccb(ap->ap_ccb_err);
742
743 if (ap->ap_ccbs) {
744 while ((ccb = ahci_get_ccb(ap)) != NULL((void *)0))
745 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (ccb
->ccb_dmamap))
;
746 free(ap->ap_ccbs, M_DEVBUF2, sc->sc_ncmds * sizeof(*ccb));
747 }
748
749 if (ap->ap_dmamem_cmd_list)
750 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
751 if (ap->ap_dmamem_rfis)
752 ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
753 if (ap->ap_dmamem_cmd_table)
754 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
755 if (ap->ap_err_scratch)
756 dma_free(ap->ap_err_scratch, DEV_BSIZE(1 << 9));
757
758 /* bus_space(9) says we dont free the subregions handle */
759
760 free(ap, M_DEVBUF2, sizeof(*ap));
761 sc->sc_ports[port] = NULL((void *)0);
762}
763
764int
765ahci_port_init(struct ahci_softc *sc, u_int port)
766{
767 struct ahci_port *ap;
768 u_int64_t dva;
769 u_int32_t cmd;
770 int rc = ENOMEM12;
771
772 ap = sc->sc_ports[port];
773#ifdef AHCI_DEBUG
774 snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
775 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
776#endif
777
778 /* Disable port interrupts */
779 ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
780
781 /* Sec 10.1.2 - deinitialise port if it is already running */
782 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
783 if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |((cmd) & (((1<<0) | (1<<15) | (1<<4) | (
1<<14))))
784 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR))((cmd) & (((1<<0) | (1<<15) | (1<<4) | (
1<<14))))
||
785 ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) {
786 int r;
787
788 r = ahci_port_stop(ap, 1);
789 if (r) {
790 printf("%s: unable to disable %s, ignoring port %d\n",
791 DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port);
792 rc = ENXIO6;
793 goto reterr;
794 }
795
796 /* Write DET to zero */
797 ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
798 }
799
800 /* Setup RFIS base address */
801 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void *)(ap->ap_dmamem_rfis)->adm_kva);
802 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0
].ds_addr)
;
803 ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32));
804 ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva);
805
806 /* Enable FIS reception and activate port. */
807 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
808 cmd |= AHCI_PREG_CMD_FRE(1<<4) | AHCI_PREG_CMD_POD(1<<2) | AHCI_PREG_CMD_SUD(1<<1);
809 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd | AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
810
811 /* Check whether port activated. Skip it if not. */
812 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
813 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
814 rc = ENXIO6;
815 goto reterr;
816 }
817
818 /* Setup command list base address */
819 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs
[0].ds_addr)
;
820 ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32));
821 ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva);
822
823 /* Wait for ICC change to complete */
824 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1));
825
826 /* Reset port */
827 rc = ahci_port_portreset(ap, 1);
828 switch (rc) {
829 case ENODEV19:
830 switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
831 case AHCI_PREG_SSTS_DET_DEV_NE0x1:
832 printf("%s: device not communicating on port %d\n",
833 DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
834 break;
835 case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4:
836 printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
837 port);
838 break;
839 default:
840 DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
841 "on port %d\n", DEVNAME(sc), port);
842 break;
843 }
844 goto reterr;
845
846 case EBUSY16:
847 printf("%s: device on port %d didn't come ready, "
848 "TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port,
849 ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
850
851 /* Try a soft reset to clear busy */
852 rc = ahci_port_softreset(ap);
853 if (rc) {
854 printf("%s: unable to communicate "
855 "with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
856 goto reterr;
857 }
858 break;
859
860 default:
861 break;
862 }
863 DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
864 DEVNAME(sc), port);
865
866 if (ap->ap_pmp_ports > 0) {
867 int p;
868
869 for (p = 0; p < ap->ap_pmp_ports; p++) {
870 int sig;
871
872 /* might need to do a portreset first here? */
873
874 /* softreset the port */
875 if (ahci_pmp_port_softreset(ap, p)) {
876 printf("%s.%d: unable to probe PMP port due to"
877 " softreset failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p);
878 continue;
879 }
880
881 sig = ahci_port_signature(ap);
882 printf("%s.%d: port signature returned %d\n",
883 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p, sig);
884 }
885 }
886
887 /* Enable command transfers on port */
888 if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
889 printf("%s: failed to start command DMA on port %d, "
890 "disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
891 rc = ENXIO6; /* couldn't start port */
892 }
893
894 /* Flush interrupts for port */
895 ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
896 ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
897
898 ahci_enable_interrupts(ap);
899
900reterr:
901 return (rc);
902}
903
904int
905ahci_default_port_start(struct ahci_port *ap, int fre_only)
906{
907 u_int32_t r;
908
909 /* Turn on FRE (and ST) */
910 r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
911 r |= AHCI_PREG_CMD_FRE(1<<4);
912 if (!fre_only)
913 r |= AHCI_PREG_CMD_ST(1<<0);
914 ahci_pwrite(ap, AHCI_PREG_CMD0x18, r);
915
916#ifdef AHCI_COALESCE
917 /* (Re-)enable coalescing on the port. */
918 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
919 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
920 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018,
921 ap->ap_sc->sc_ccc_ports_cur);
922 }
923#endif
924
925 return (0);
926}
927
928int
929ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
930{
931 u_int32_t r;
932
933#ifdef AHCI_COALESCE
934 /* Disable coalescing on the port while it is stopped. */
935 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
936 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
937 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018,
938 ap->ap_sc->sc_ccc_ports_cur);
939 }
940#endif
941
942 /* Turn off ST (and FRE) */
943 r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
944 r &= ~AHCI_PREG_CMD_ST(1<<0);
945 if (stop_fis_rx)
946 r &= ~AHCI_PREG_CMD_FRE(1<<4);
947 ahci_pwrite(ap, AHCI_PREG_CMD0x18, r);
948
949 /* Wait for CR to go off */
950 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)ahci_pwait_eq((ap), (0x18), ((1<<15)), 0, (1)))
951 return (1);
952
953 /* Wait for FR to go off */
954 if (stop_fis_rx &&
955 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)ahci_pwait_eq((ap), (0x18), ((1<<14)), 0, (1)))
956 return (2);
957
958 return (0);
959}
960
961/* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
962int
963ahci_port_clo(struct ahci_port *ap)
964{
965 struct ahci_softc *sc = ap->ap_sc;
966 u_int32_t cmd;
967
968 /* Only attempt CLO if supported by controller */
969 if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(sc, 0x000)) & ((1<<24))))
970 return (1);
971
972 /* Issue CLO */
973 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
974#ifdef DIAGNOSTIC1
975 if (ISSET(cmd, AHCI_PREG_CMD_ST)((cmd) & ((1<<0))))
976 printf("%s: CLO requested while port running\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
977#endif
978 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd | AHCI_PREG_CMD_CLO(1<<3));
979
980 /* Wait for completion */
981 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)ahci_pwait_eq((ap), (0x18), ((1<<3)), 0, (1))) {
982 printf("%s: CLO did not complete\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
983 return (1);
984 }
985
986 return (0);
987}
988
989/* AHCI soft reset, Section 10.4.1 */
990int
991ahci_port_softreset(struct ahci_port *ap)
992{
993 struct ahci_ccb *ccb = NULL((void *)0);
994 struct ahci_cmd_hdr *cmd_slot;
995 u_int8_t *fis;
996 int s, rc = EIO5, oldstate;
997 u_int32_t cmd;
998
999 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
1000
1001 s = splbio()splraise(0x3);
1002 oldstate = ap->ap_state;
1003 ap->ap_state = AP_S_ERROR_RECOVERY3;
1004
1005 /* Save previous command register state */
1006 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1007
1008 /* Idle port */
1009 if (ahci_port_stop(ap, 0)) {
1010 printf("%s: failed to stop port, cannot softreset\n",
1011 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1012 goto err;
1013 }
1014
1015 /* Request CLO if device appears hung */
1016 if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |((ahci_pread(ap, 0x20)) & ((1<<7) | (1<<3)))
1017 AHCI_PREG_TFD_STS_DRQ)((ahci_pread(ap, 0x20)) & ((1<<7) | (1<<3))))
1018 ahci_port_clo(ap);
1019
1020 /* Clear port errors to permit TFD transfer */
1021 ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30));
1022
1023 /* Restart port */
1024 if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
1025 printf("%s: failed to start port, cannot softreset\n",
1026 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1027 goto err;
1028 }
1029
1030 /* Check whether CLO worked */
1031 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0,
(1))
1032 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0,
(1))
) {
1033 printf("%s: CLO %s, need port reset\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1034 ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(ap->ap_sc, 0x000)) & ((1<<24)))
1035 ? "failed" : "unsupported");
1036 rc = EBUSY16;
1037 goto err;
1038 }
1039
1040 /* Prep first D2H command with SRST feature & clear busy/reset flags */
1041 ccb = ahci_get_err_ccb(ap);
1042 cmd_slot = ccb->ccb_cmd_hdr;
1043 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
1044
1045 fis = ccb->ccb_cmd_table->cfis;
1046 fis[0] = ATA_FIS_TYPE_H2D0x27;
1047 fis[15] = ATA_FIS_CONTROL_SRST0x04;
1048
1049 cmd_slot->prdtl = 0;
1050 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | (1<<6))))
1051 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | (1<<6))))
1052 AHCI_CMD_LIST_FLAG_W)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | (1<<6))))
;
1053
1054 ccb->ccb_xa.state = ATA_S_PENDING1;
1055 if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0)
1056 goto err;
1057
1058 /* Prep second D2H command to read status and complete reset sequence */
1059 fis[0] = ATA_FIS_TYPE_H2D0x27;
1060 fis[15] = 0;
1061
1062 cmd_slot->prdtl = 0;
1063 htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<6))))
;
1064
1065 ccb->ccb_xa.state = ATA_S_PENDING1;
1066 if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0)
1067 goto err;
1068
1069 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1
<<0)), 0, (1))
1070 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1
<<0)), 0, (1))
) {
1071 printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1072 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ahci_pread(ap, AHCI_PREG_TFD0x20),
1073 AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
1074 rc = EBUSY16;
1075 goto err;
1076 }
1077
1078 rc = 0;
1079err:
1080 if (ccb != NULL((void *)0)) {
1081 /* Abort our command, if it failed, by stopping command DMA. */
1082 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) {
1083 printf("%s: stopping the port, softreset slot %d was "
1084 "still active.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot);
1085 ahci_port_stop(ap, 0);
1086 }
1087 ccb->ccb_xa.state = ATA_S_ERROR3;
1088 ahci_put_err_ccb(ccb);
1089 }
1090
1091 /* Restore saved CMD register state */
1092 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1093 ap->ap_state = oldstate;
1094
1095 splx(s)spllower(s);
1096
1097 return (rc);
1098}
1099
1100int
1101ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
1102{
1103 struct ahci_ccb *ccb = NULL((void *)0);
1104 u_int32_t data;
1105 int count;
1106 int rc;
1107 int s;
1108 struct ahci_cmd_hdr *cmd_slot;
1109 u_int8_t *fis;
1110
1111 s = splbio()splraise(0x3);
1112 /* ignore spurious IFS errors while resetting */
1113 DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
1114 ap->ap_pmp_ignore_ifs = 1;
1115
1116 count = 2;
1117 rc = 0;
1118 do {
1119 if (ccb != NULL((void *)0)) {
1120 ahci_put_pmp_ccb(ccb);
1121 ccb = NULL((void *)0);
1122 }
1123
1124 if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1125 printf("%s.%d: unable to clear PHY status\n",
1126 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1127 }
1128 ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1129 /* maybe don't do this on the first loop: */
1130 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1131 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1132
1133 /* send first softreset FIS */
1134 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1135 cmd_slot = ccb->ccb_cmd_hdr;
1136 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
1137
1138 fis = ccb->ccb_cmd_table->cfis;
1139 fis[0] = ATA_FIS_TYPE_H2D0x27;
1140 fis[1] = pmp_port;
1141 fis[15] = ATA_FIS_CONTROL_SRST0x04 | ATA_FIS_CONTROL_4BIT0x08;
1142
1143 cmd_slot->prdtl = 0;
1144 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | (pmp_port << 12))))
1145 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | (pmp_port << 12))))
1146 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | (pmp_port << 12))))
;
1147
1148 ccb->ccb_xa.state = ATA_S_PENDING1;
1149
1150 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
1151 PORTNAME(ap), pmp_port);
1152 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1153 printf("%s.%d: PMP port softreset cmd failed\n",
1154 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1155 rc = EBUSY16;
1156 if (count > 0) {
1157 /* probably delay a while to allow
1158 * it to settle down?
1159 */
1160 }
1161 continue;
1162 }
1163
1164 /* send signature FIS */
1165 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
1166 fis[0] = ATA_FIS_TYPE_H2D0x27;
1167 fis[1] = pmp_port;
1168 fis[15] = ATA_FIS_CONTROL_4BIT0x08;
1169
1170 cmd_slot->prdtl = 0;
1171 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(pmp_port << 12))))
1172 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(pmp_port << 12))))
;
1173
1174 DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
1175 PORTNAME(ap), pmp_port);
1176 ccb->ccb_xa.state = ATA_S_PENDING1;
1177 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1178 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
1179 "failed\n", PORTNAME(ap), pmp_port);
1180 rc = EBUSY16;
1181 if (count > 0) {
1182 /* sleep a while? */
1183 }
1184 continue;
1185 }
1186
1187 fis[15] = 0;
1188 break;
1189 } while (count--);
1190
1191 if (ccb != NULL((void *)0)) {
1192 ahci_put_pmp_ccb(ccb);
1193 ccb = NULL((void *)0);
1194 }
1195
1196 /* clean up a bit */
1197 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1198 ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1199 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1200 ap->ap_pmp_ignore_ifs = 0;
1201 DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
1202 splx(s)spllower(s);
1203
1204 return (rc);
1205}
1206
1207int
1208ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
1209{
1210 int sig;
1211
1212 ap->ap_state = AP_S_PMP_PORT_PROBE2;
1213
1214 DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
1215 pmp_port);
1216 if (ahci_pmp_port_portreset(ap, pmp_port)) {
1217 printf("%s.%d: unable to probe PMP port; portreset failed\n",
1218 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1219 ap->ap_state = AP_S_NORMAL0;
1220 return (ATA_PORT_T_NONE0);
1221 }
1222
1223 if (ahci_pmp_port_softreset(ap, pmp_port)) {
1224 printf("%s.%d: unable to probe PMP port due to softreset "
1225 "failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1226 ap->ap_state = AP_S_NORMAL0;
1227 return (ATA_PORT_T_NONE0);
1228 }
1229
1230 sig = ahci_port_signature(ap);
1231 DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
1232 PORTNAME(ap), pmp_port, sig);
1233 ap->ap_state = AP_S_NORMAL0;
1234 return (sig);
1235}
1236
1237
1238void
1239ahci_flush_tfd(struct ahci_port *ap)
1240{
1241 u_int32_t r;
1242
1243 r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1244 if (r & AHCI_PREG_SERR_DIAG_X(1<<10))
1245 ahci_pwrite(ap, AHCI_PREG_SERR0x30, AHCI_PREG_SERR_DIAG_X(1<<10));
1246}
1247
1248u_int32_t
1249ahci_active_mask(struct ahci_port *ap)
1250{
1251 u_int32_t mask;
1252
1253 mask = ahci_pread(ap, AHCI_PREG_CI0x38);
1254 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30))
1255 mask |= ahci_pread(ap, AHCI_PREG_SACT0x34);
1256 return mask;
1257}
1258
1259void
1260ahci_pmp_probe_timeout(void *cookie)
1261{
1262 struct ahci_ccb *ccb = cookie;
1263 struct ahci_port *ap = ccb->ccb_port;
1264 u_int32_t mask;
1265
1266 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
1267 switch (ccb->ccb_xa.state) {
1268 case ATA_S_PENDING1:
1269 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb)
->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry
.tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = (
ccb)->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = (
ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) =
((void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1);
} while (0)
;
1270 ccb->ccb_xa.state = ATA_S_TIMEOUT4;
1271 break;
1272
1273 case ATA_S_ONCHIP5:
1274 case ATA_S_ERROR3: /* currently mostly here for the ATI SBx00 quirk */
1275 /* clear the command on-chip */
1276 KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&((ap->ap_active == (1 << ccb->ccb_slot) &&
ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0"
))
1277 ap->ap_sactive == 0)((ap->ap_active == (1 << ccb->ccb_slot) &&
ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0"
))
;
1278 ahci_port_stop(ap, 0);
1279 ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
1280
1281 if (ahci_active_mask(ap) != 0) {
1282 ahci_port_stop(ap, 0);
1283 ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
1284 mask = ahci_active_mask(ap);
1285 if (mask != 0) {
1286 printf("%s: ahci_pmp_probe_timeout: failed to "
1287 "clear active cmds: %08x\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1288 mask);
1289 }
1290 }
1291
1292 ccb->ccb_xa.state = ATA_S_TIMEOUT4;
1293 ap->ap_active &= ~(1 << ccb->ccb_slot);
1294 KASSERT(ap->ap_active_cnt > 0)((ap->ap_active_cnt > 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 1294, "ap->ap_active_cnt > 0"
))
;
1295 --ap->ap_active_cnt;
1296 DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, "
1297 "active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot,
1298 ap->ap_active, ap->ap_active_cnt);
1299 break;
1300
1301 default:
1302 panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
1303 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state);
1304 }
1305}
1306
1307int
1308ahci_port_signature(struct ahci_port *ap)
1309{
1310 u_int32_t sig;
1311
1312 sig = ahci_pread(ap, AHCI_PREG_SIG0x24);
1313 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI0xeb140101 & 0xffff0000))
1314 return (ATA_PORT_T_ATAPI2);
1315 else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER0x96690101 &
1316 0xffff0000))
1317 return (ATA_PORT_T_PM3);
1318 else
1319 return (ATA_PORT_T_DISK1);
1320}
1321
1322int
1323ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
1324{
1325 u_int32_t cmd, data;
1326 int loop;
1327 int rc = 1;
1328 int s;
1329
1330 s = splbio()splraise(0x3);
1331 DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
1332 pmp_port);
1333
1334 /* Save previous command register state */
1335 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1336
1337 /* turn off power management and disable the PHY */
1338 data = AHCI_PREG_SCTL_IPM_DISABLED0x300;
1339 /* maybe add AHCI_PREG_SCTL_DET_DISABLE */
1340 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1))
1341 goto err;
1342 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1343 goto err;
1344 delay(10000)(*delay_func)(10000);
1345
1346 /* start COMRESET */
1347 data = AHCI_PREG_SCTL_IPM_DISABLED0x300 | AHCI_PREG_SCTL_DET_INIT0x1;
1348 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1349 DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
1350 pmp_port);
1351 data |= AHCI_PREG_SCTL_SPD_GEN10x10;
1352 } else
1353 data |= AHCI_PREG_SCTL_SPD_ANY0x00;
1354
1355 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1356 goto err;
1357
1358 /* give it a while to settle down */
1359 delay(100000)(*delay_func)(100000);
1360
1361 if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1362 printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1363 pmp_port);
1364 }
1365
1366 /* start trying to negotiate */
1367 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1368 data = AHCI_PREG_SCTL_IPM_DISABLED0x300 | AHCI_PREG_SCTL_DET_NONE0x0;
1369 if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1370 goto err;
1371
1372 /* give it a while to detect */
1373 for (loop = 3; loop; --loop) {
1374 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data))
1375 goto err;
1376 if (data & AHCI_PREG_SSTS_DET0xf)
1377 break;
1378 delay(100000)(*delay_func)(100000);
1379 }
1380 if (loop == 0) {
1381 printf("%s.%d: port is unplugged\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1382 goto err;
1383 }
1384
1385 /* give it even longer to fully negotiate */
1386 for (loop = 30; loop; --loop) {
1387 if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data))
1388 goto err;
1389 if ((data & AHCI_PREG_SSTS_DET0xf) == AHCI_PREG_SSTS_DET_DEV0x3)
1390 break;
1391 delay(100000)(*delay_func)(100000);
1392 }
1393
1394 if (loop == 0) {
1395 printf("%s.%d: device is not negotiating\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1396 pmp_port);
1397 goto err;
1398 }
1399
1400 /* device detected */
1401 DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
1402 pmp_port);
1403
1404 /* clean up a bit */
1405 delay(100000)(*delay_func)(100000);
1406 ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1407 ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1408 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1409
1410 rc = 0;
1411err:
1412 /* Restore preserved port state */
1413 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1414 splx(s)spllower(s);
1415 return (rc);
1416}
1417
1418/* AHCI port reset, Section 10.4.2 */
1419
1420void
1421ahci_port_comreset(struct ahci_port *ap)
1422{
1423 u_int32_t r;
1424
1425 r = AHCI_PREG_SCTL_IPM_DISABLED0x300 | AHCI_PREG_SCTL_DET_INIT0x1;
1426 if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1427 DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
1428 r |= AHCI_PREG_SCTL_SPD_GEN10x10;
1429 } else
1430 r |= AHCI_PREG_SCTL_SPD_ANY0x00;
1431 ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r);
1432 delay(10000)(*delay_func)(10000); /* wait at least 1ms for COMRESET to be sent */
1433 r &= ~AHCI_PREG_SCTL_DET_INIT0x1;
1434 r |= AHCI_PREG_SCTL_DET_NONE0x0;
1435 ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r);
1436 delay(10000)(*delay_func)(10000);
1437}
1438
1439void
1440ahci_port_portreset_start(struct ahci_port *ap)
1441{
1442 int s;
1443
1444 s = splbio()splraise(0x3);
1445 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1446
1447 /* Save previous command register state */
1448 ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1449
1450 /* Clear ST, ignoring failure */
1451 ahci_port_stop(ap, 0);
1452
1453 /* Perform device detection */
1454 ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
1455 delay(10000)(*delay_func)(10000);
1456 ahci_port_comreset(ap);
1457 splx(s)spllower(s);
1458}
1459
1460int
1461ahci_port_portreset_poll(struct ahci_port *ap)
1462{
1463 if ((ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) !=
1464 AHCI_PREG_SSTS_DET_DEV0x3)
1465 return (EAGAIN35);
1466 return (0);
1467}
1468
1469void
1470ahci_port_portreset_wait(struct ahci_port *ap)
1471{
1472 int i;
1473
1474 for (i = 0; i < 1000; i++) {
1475 if (ahci_port_portreset_poll(ap) == 0)
1476 break;
1477 delay(1000)(*delay_func)(1000);
1478 }
1479}
1480
1481int
1482ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
1483{
1484 int rc, s, retries = 0;
1485
1486 s = splbio()splraise(0x3);
1487retry:
1488 if (ahci_port_portreset_poll(ap)) {
1489 rc = ENODEV19;
1490 if (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
1491 /* this may be a port multiplier with no device
1492 * on port 0, so still do the pmp check if requested.
1493 */
1494 } else {
1495 goto err;
1496 }
1497 } else {
1498 /* Clear SERR (incl X bit), so TFD can update */
1499 ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30));
1500
1501 /* Wait for device to become ready */
1502 if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1
<<0)), 0, (3))
1503 AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3) | (1
<<0)), 0, (3))
) {
1504 /* even if the device doesn't wake up, check if there's
1505 * a port multiplier there
1506 */
1507 if (retries == 0) {
1508 retries = 1;
1509 ahci_port_comreset(ap);
1510 ahci_port_portreset_wait(ap);
1511 goto retry;
1512 }
1513 rc = EBUSY16;
1514 } else {
1515 rc = 0;
1516 }
1517 }
1518
1519 if (pmp != 0) {
1520 if (ahci_port_detect_pmp(ap)) {
1521 /* reset again without pmp support */
1522 pmp = 0;
1523 retries = 0;
1524 ahci_port_comreset(ap);
1525 ahci_port_portreset_wait(ap);
1526 goto retry;
1527 }
1528 }
1529
1530err:
1531 /* Restore preserved port state */
1532 ahci_pwrite(ap, AHCI_PREG_CMD0x18, ap->ap_saved_cmd);
1533 ap->ap_saved_cmd = 0;
1534 splx(s)spllower(s);
1535
1536 return (rc);
1537}
1538
1539int
1540ahci_port_portreset(struct ahci_port *ap, int pmp)
1541{
1542 ahci_port_portreset_start(ap);
1543 ahci_port_portreset_wait(ap);
1544 return (ahci_port_portreset_finish(ap, pmp));
1545}
1546
1547int
1548ahci_port_detect_pmp(struct ahci_port *ap)
1549{
1550 int count, pmp_rc, rc;
1551 u_int32_t r, cmd;
1552 struct ahci_cmd_hdr *cmd_slot;
1553 struct ahci_ccb *ccb = NULL((void *)0);
1554 u_int8_t *fis = NULL((void *)0);
1555
1556 if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP(1<<2)) ||
1557 !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)((ahci_read(ap->ap_sc, 0x000)) & ((1<<17)))) {
1558 return 0;
1559 }
1560
1561 rc = 0;
1562 pmp_rc = 0;
1563 count = 2;
1564 do {
1565 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
1566 count);
1567 if (ccb != NULL((void *)0)) {
1568 ahci_put_pmp_ccb(ccb);
1569 ccb = NULL((void *)0);
1570 }
1571 ahci_port_stop(ap, 0);
1572 ap->ap_state = AP_S_PMP_PROBE1;
1573
1574 /* set PMA in cmd reg */
1575 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1576 if ((cmd & AHCI_PREG_CMD_PMA(1<<17)) == 0) {
1577 cmd |= AHCI_PREG_CMD_PMA(1<<17);
1578 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1579 }
1580
1581 /* Flush errors and request CLO unconditionally,
1582 * then start the port
1583 */
1584 r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1585 if (r & AHCI_PREG_SERR_DIAG_X(1<<10))
1586 ahci_pwrite(ap, AHCI_PREG_SERR0x30,
1587 AHCI_PREG_SERR_DIAG_X(1<<10));
1588
1589 /* Request CLO */
1590 ahci_port_clo(ap);
1591
1592 /* Clear port errors to permit TFD transfer */
1593 r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1594 ahci_pwrite(ap, AHCI_PREG_SERR0x30, r);
1595
1596 /* Restart port */
1597 if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
1598 rc = EBUSY16;
1599 printf("%s: failed to start port, cannot probe PMP\n",
1600 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1601 break;
1602 }
1603
1604 /* Check whether CLO worked */
1605 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0,
(1))
1606 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0,
(1))
) {
1607 u_int32_t cap;
1608
1609 cap = ahci_read(ap->ap_sc, AHCI_REG_CAP0x000);
1610 printf("%s: CLO %s, need port reset\n",
1611 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1612 ISSET(cap, AHCI_REG_CAP_SCLO)((cap) & ((1<<24)))
1613 ? "failed" : "unsupported");
1614 pmp_rc = EBUSY16;
1615 break;
1616 }
1617
1618 /* Prep first command with SRST feature &
1619 * clear busy/reset flags
1620 */
1621 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1622 cmd_slot = ccb->ccb_cmd_hdr;
1623 memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
1624 sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
1625
1626 fis = ccb->ccb_cmd_table->cfis;
1627 fis[0] = ATA_FIS_TYPE_H2D0x27;
1628 fis[1] = SATA_PMP_CONTROL_PORT0x0f;
1629 fis[15] = ATA_FIS_CONTROL_SRST0x04 | ATA_FIS_CONTROL_4BIT0x08;
1630
1631 cmd_slot->prdtl = 0;
1632 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | 0xf000)))
1633 AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | 0xf000)))
1634 AHCI_CMD_LIST_FLAG_PMP)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(1<<10) | (1<<8) | 0xf000)))
;
1635
1636 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
1637 PORTNAME(ap));
1638 ccb->ccb_xa.state = ATA_S_PENDING1;
1639 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1640 DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
1641 PORTNAME(ap));
1642 pmp_rc = EBUSY16;
1643 continue;
1644 }
1645
1646 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0,
(1))
1647 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) | (1<<3)), 0,
(1))
) {
1648 printf("%s: port busy after first PMP probe FIS\n",
1649 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1650 }
1651
1652 /* clear errors in case the device
1653 * didn't reset cleanly
1654 */
1655 ahci_flush_tfd(ap);
1656 r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1657 ahci_pwrite(ap, AHCI_PREG_SERR0x30, r);
1658
1659 /* Prep second command to read status and
1660 * complete reset sequence
1661 */
1662 memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
1663 sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
1664 fis[0] = ATA_FIS_TYPE_H2D0x27;
1665 fis[1] = SATA_PMP_CONTROL_PORT0x0f;
1666 fis[15] = ATA_FIS_CONTROL_4BIT0x08;
1667
1668 cmd_slot->prdtl = 0;
1669 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
0xf000)))
1670 AHCI_CMD_LIST_FLAG_PMP)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
0xf000)))
;
1671
1672 DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
1673 PORTNAME(ap));
1674 ccb->ccb_xa.state = ATA_S_PENDING1;
1675 if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1676 DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
1677 "cmd failed\n", PORTNAME(ap));
1678 pmp_rc = EBUSY16;
1679 continue;
1680 }
1681
1682 /* apparently we need to retry at least once
1683 * to get the right signature
1684 */
1685 fis[15] = 0;
1686 pmp_rc = 0;
1687 } while (--count);
1688
1689 if (ccb != NULL((void *)0)) {
1690 ahci_put_pmp_ccb(ccb);
1691 ccb = NULL((void *)0);
1692 }
1693
1694 if (ap->ap_state == AP_S_PMP_PROBE1) {
1695 ap->ap_state = AP_S_NORMAL0;
1696 }
1697
1698 if (pmp_rc == 0) {
1699 if (ahci_port_signature(ap) != ATA_PORT_T_PM3) {
1700 DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
1701 PORTNAME(ap));
1702 pmp_rc = EBUSY16;
1703 } else {
1704 DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
1705 PORTNAME(ap));
1706 }
1707 }
1708
1709 if (pmp_rc == 0) {
1710 if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
1711 pmp_rc = EBUSY16;
1712 } else {
1713 rc = 0;
1714 }
1715 }
1716
1717 /* if PMP detection failed, so turn off the PMA bit and
1718 * reset the port again
1719 */
1720 if (pmp_rc != 0) {
1721 DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
1722 "the port\n", PORTNAME(ap));
1723 ahci_port_stop(ap, 0);
1724 ahci_port_clo(ap);
1725 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1726 cmd &= ~AHCI_PREG_CMD_PMA(1<<17);
1727 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1728
1729 ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
1730 ahci_port_stop(ap, 0);
1731 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29))
1732 ahci_pwrite(ap, AHCI_PREG_SNTF0x3c, -1);
1733 ahci_flush_tfd(ap);
1734 ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1735
1736 ahci_pwrite(ap, AHCI_PREG_IS0x10, -1);
1737
1738 ahci_enable_interrupts(ap);
1739
1740 rc = pmp_rc;
1741 }
1742
1743 return (rc);
1744}
1745
1746void
1747ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
1748 u_int32_t flags)
1749{
1750 flags |= len - 1;
1751
1752 htolem64(&prd->dba, addr)(*(__uint64_t *)(&prd->dba) = ((__uint64_t)(addr)));
1753 htolem32(&prd->flags, flags)(*(__uint32_t *)(&prd->flags) = ((__uint32_t)(flags)));
1754}
1755
1756int
1757ahci_load_prdt(struct ahci_ccb *ccb)
1758{
1759 struct ahci_port *ap = ccb->ccb_port;
1760 struct ahci_softc *sc = ap->ap_sc;
1761 struct ata_xfer *xa = &ccb->ccb_xa;
1762 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
1763 bus_dmamap_t dmap = ccb->ccb_dmamap;
1764 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1765 int i, error;
1766
1767 if (xa->datalen == 0) {
1768 ccb->ccb_cmd_hdr->prdtl = 0;
1769 return (0);
1770 }
1771
1772 error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap)
, (xa->data), (xa->datalen), (((void *)0)), ((xa->flags
& (1<<2)) ? 0x0001 : 0x0000))
1773 (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK)(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap)
, (xa->data), (xa->datalen), (((void *)0)), ((xa->flags
& (1<<2)) ? 0x0001 : 0x0000))
;
1774 if (error != 0) {
1775 printf("%s: error %d loading dmamap\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), error);
1776 return (1);
1777 }
1778
1779 for (i = 0; i < dmap->dm_nsegs - 1; i++) {
1780 ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
1781 dmap->dm_segs[i].ds_len, 0);
1782 }
1783
1784 ahci_load_prdt_seg(&prdt[i],
1785 dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
1786 ISSET(xa->flags, ATA_F_PIO)((xa->flags) & ((1<<4))) ? AHCI_PRDT_FLAG_INTR(1<<31) : 0);
1787
1788 htolem16(&cmd_slot->prdtl, dmap->dm_nsegs)(*(__uint16_t *)(&cmd_slot->prdtl) = ((__uint16_t)(dmap
->dm_nsegs)))
;
1789
1790 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap)
, (0), (dmap->dm_mapsize), ((xa->flags & (1<<
0)) ? 0x01 : 0x04))
1791 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap)
, (0), (dmap->dm_mapsize), ((xa->flags & (1<<
0)) ? 0x01 : 0x04))
1792 BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap)
, (0), (dmap->dm_mapsize), ((xa->flags & (1<<
0)) ? 0x01 : 0x04))
;
1793
1794 return (0);
1795}
1796
1797void
1798ahci_unload_prdt(struct ahci_ccb *ccb)
1799{
1800 struct ahci_port *ap = ccb->ccb_port;
1801 struct ahci_softc *sc = ap->ap_sc;
1802 struct ata_xfer *xa = &ccb->ccb_xa;
1803 bus_dmamap_t dmap = ccb->ccb_dmamap;
1804
1805 if (xa->datalen != 0) {
1806 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap)
, (0), (dmap->dm_mapsize), ((xa->flags & (1<<
0)) ? 0x02 : 0x08))
1807 (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap)
, (0), (dmap->dm_mapsize), ((xa->flags & (1<<
0)) ? 0x02 : 0x08))
1808 BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap)
, (0), (dmap->dm_mapsize), ((xa->flags & (1<<
0)) ? 0x02 : 0x08))
;
1809
1810 bus_dmamap_unload(sc->sc_dmat, dmap)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (dmap
))
;
1811
1812 if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6))
1813 xa->resid = 0;
1814 else
1815 xa->resid = xa->datalen -
1816 lemtoh32(&ccb->ccb_cmd_hdr->prdbc)((__uint32_t)(*(__uint32_t *)(&ccb->ccb_cmd_hdr->prdbc
)))
;
1817 }
1818}
1819
1820int
1821ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
1822{
1823 struct ahci_port *ap = ccb->ccb_port;
1824 int s;
1825
1826 s = splbio()splraise(0x3);
1827 ahci_start(ccb);
1828 do {
1829 if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot
))
1830 1 << ccb->ccb_slot)((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot
))
) {
1831 splx(s)spllower(s);
1832 return (0);
1833 }
1834 if (ccb->ccb_xa.state == ATA_S_ERROR3) {
1835 DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
1836 PORTNAME(ap), ccb->ccb_slot);
1837 /* pretend it timed out? */
1838 if (timeout_fn != NULL((void *)0)) {
1839 timeout_fn(ccb);
1840 }
1841 splx(s)spllower(s);
1842 return (1);
1843 }
1844
1845 delay(1000)(*delay_func)(1000);
1846 } while (--timeout > 0);
1847
1848 /* Run timeout while at splbio, otherwise ahci_intr could interfere. */
1849 if (timeout_fn != NULL((void *)0))
1850 timeout_fn(ccb);
1851
1852 splx(s)spllower(s);
1853
1854 return (1);
1855}
1856
1857void
1858ahci_start(struct ahci_ccb *ccb)
1859{
1860 struct ahci_port *ap = ccb->ccb_port;
1861 struct ahci_softc *sc = ap->ap_sc;
1862
1863 /* Zero transferred byte count before transfer */
1864 ccb->ccb_cmd_hdr->prdbc = 0;
1865
1866 /* Sync command list entry and corresponding command table entry */
1867 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04)
)
1868 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04)
)
1869 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04)
)
;
1870 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04
))
1871 ccb->ccb_slot * sizeof(struct ahci_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04
))
1872 sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04
))
;
1873
1874 /* Prepare RFIS area for write by controller */
1875 bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis)
), (0x01))
1876 sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis)
), (0x01))
;
1877
1878 if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) {
1879 /* Issue NCQ commands only when there are no outstanding
1880 * standard commands. */
1881 if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending)(((&ap->ap_ccb_pending)->tqh_first) == ((void *)0)) ||
1882 (ap->ap_sactive != 0 &&
1883 ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
1884 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry
.tqe_prev = (&ap->ap_ccb_pending)->tqh_last; *(&
ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending
)->tqh_last = &(ccb)->ccb_entry.tqe_next; } while (
0)
;
1885 } else {
1886 KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 1886, "ap->ap_active_cnt == 0"
))
;
1887 ap->ap_sactive |= (1 << ccb->ccb_slot);
1888 ccb->ccb_xa.state = ATA_S_ONCHIP5;
1889 ahci_pwrite(ap, AHCI_PREG_SACT0x34, 1 << ccb->ccb_slot);
1890 ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
1891 ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
1892 }
1893 } else {
1894 /* Wait for all NCQ commands to finish before issuing standard
1895 * command. */
1896 if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
1897 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry
.tqe_prev = (&ap->ap_ccb_pending)->tqh_last; *(&
ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending
)->tqh_last = &(ccb)->ccb_entry.tqe_next; } while (
0)
;
1898 else if (ap->ap_active_cnt < 2) {
1899 ap->ap_active |= 1 << ccb->ccb_slot;
1900 ccb->ccb_xa.state = ATA_S_ONCHIP5;
1901 ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
1902 ap->ap_active_cnt++;
1903 }
1904 }
1905}
1906
1907void
1908ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1909{
1910 struct ahci_ccb *nextccb;
1911 u_int32_t sact_change = 0;
1912
1913 KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 1913, "ap->ap_active_cnt == 0"
))
;
1914
1915 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1916 if (nextccb == NULL((void *)0) || !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)))
1917 return;
1918
1919 /* Start all the NCQ commands at the head of the pending list.
1920 * If a port multiplier is attached to the port, we can only
1921 * issue commands for one of its ports at a time.
1922 */
1923 if (ap->ap_sactive != 0 &&
1924 ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
1925 return;
1926 }
1927
1928 ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
1929 do {
1930 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void *)0)) (
nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb
)->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)->
tqh_last = (nextccb)->ccb_entry.tqe_prev; *(nextccb)->ccb_entry
.tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)->
ccb_entry.tqe_prev) = ((void *)-1); ((nextccb)->ccb_entry.
tqe_next) = ((void *)-1); } while (0)
;
1931 sact_change |= 1 << nextccb->ccb_slot;
1932 nextccb->ccb_xa.state = ATA_S_ONCHIP5;
1933 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1934 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) &&
1935 (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
1936
1937 ap->ap_sactive |= sact_change;
1938 ahci_pwrite(ap, AHCI_PREG_SACT0x34, sact_change);
1939 ahci_pwrite(ap, AHCI_PREG_CI0x38, sact_change);
1940}
1941
1942void
1943ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
1944{
1945 struct ahci_ccb *nextccb;
1946
1947 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1948 if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6))) {
1949 if (last_was_ncq) {
1950 KASSERT(nextccb->ccb_xa.pmp_port !=((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951
, "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port"))
1951 ap->ap_pmp_ncq_port)((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951
, "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port"))
;
1952 /* otherwise it should have been started already */
1953 } else {
1954 ap->ap_active_cnt--;
1955 }
1956
1957 /* Issue NCQ commands only when there are no outstanding
1958 * standard commands, and previous NCQ commands for other
1959 * PMP ports have finished.
1960 */
1961 if (ap->ap_active == 0)
1962 ahci_issue_pending_ncq_commands(ap);
1963 else
1964 KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 1964, "ap->ap_active_cnt == 1"
))
;
1965 } else if (nextccb) {
1966 if (ap->ap_sactive != 0 || last_was_ncq)
1967 KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 1967, "ap->ap_active_cnt == 0"
))
;
1968
1969 /* Wait for all NCQ commands to finish before issuing standard
1970 * command. */
1971 if (ap->ap_sactive != 0)
1972 return;
1973
1974 /* Keep up to 2 standard commands on-chip at a time. */
1975 do {
1976 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void *)0)) (
nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb
)->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)->
tqh_last = (nextccb)->ccb_entry.tqe_prev; *(nextccb)->ccb_entry
.tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)->
ccb_entry.tqe_prev) = ((void *)-1); ((nextccb)->ccb_entry.
tqe_next) = ((void *)-1); } while (0)
;
1977 ap->ap_active |= 1 << nextccb->ccb_slot;
1978 nextccb->ccb_xa.state = ATA_S_ONCHIP5;
1979 ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << nextccb->ccb_slot);
1980 if (last_was_ncq)
1981 ap->ap_active_cnt++;
1982 if (ap->ap_active_cnt == 2)
1983 break;
1984 KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 1984, "ap->ap_active_cnt == 1"
))
;
1985 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1986 } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)));
1987 } else if (!last_was_ncq) {
1988 KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2)((ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 1988, "ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2"
))
;
1989
1990 /* Standard command finished, none waiting to start. */
1991 ap->ap_active_cnt--;
1992 } else {
1993 KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 1993, "ap->ap_active_cnt == 0"
))
;
1994
1995 /* NCQ command finished. */
1996 }
1997}
1998
1999int
2000ahci_intr(void *arg)
2001{
2002 struct ahci_softc *sc = arg;
2003 u_int32_t is, ack = 0;
2004 int port;
2005
2006 /* Read global interrupt status */
2007 is = ahci_read(sc, AHCI_REG_IS0x008);
2008 if (is == 0 || is == 0xffffffff)
2009 return (0);
2010 ack = is;
2011
2012#ifdef AHCI_COALESCE
2013 /* Check coalescing interrupt first */
2014 if (is & sc->sc_ccc_mask) {
2015 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2016 DEVNAME(sc));
2017 is &= ~sc->sc_ccc_mask;
2018 is |= sc->sc_ccc_ports_cur;
2019 }
2020#endif
2021
2022 /* Process interrupts for each port */
2023 while (is) {
2024 port = ffs(is) - 1;
2025 if (sc->sc_ports[port])
2026 ahci_port_intr(sc->sc_ports[port],
2027 AHCI_PREG_CI_ALL_SLOTS0xffffffff);
2028 is &= ~(1 << port);
2029 }
2030
2031 /* Finally, acknowledge global interrupt */
2032 ahci_write(sc, AHCI_REG_IS0x008, ack);
2033
2034 return (1);
2035}
2036
2037u_int32_t
2038ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
2039{
2040 struct ahci_softc *sc = ap->ap_sc;
2041 u_int32_t is, ci_saved, ci_masked, processed = 0;
2042 int slot, need_restart = 0;
2043 int process_error = 0;
2044 struct ahci_ccb *ccb;
2045 volatile u_int32_t *active;
2046#ifdef DIAGNOSTIC1
2047 u_int32_t tmp;
2048#endif
2049
2050 is = ahci_pread(ap, AHCI_PREG_IS0x10);
2051
2052 /* Ack port interrupt only if checking all command slots. */
2053 if (ci_mask == AHCI_PREG_CI_ALL_SLOTS0xffffffff)
2054 ahci_pwrite(ap, AHCI_PREG_IS0x10, is);
2055
2056 if (is)
2057 DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
2058 is, AHCI_PFMT_IS);
2059
2060 if (ap->ap_sactive) {
2061 /* Active NCQ commands - use SActive instead of CI */
2062 KASSERT(ap->ap_active == 0)((ap->ap_active == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 2062, "ap->ap_active == 0"))
;
2063 KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2063, "ap->ap_active_cnt == 0"
))
;
2064 ci_saved = ahci_pread(ap, AHCI_PREG_SACT0x34);
2065 active = &ap->ap_sactive;
2066 } else {
2067 /* Save CI */
2068 ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38);
2069 active = &ap->ap_active;
2070 }
2071
2072 if (is & AHCI_PREG_IS_TFES(1<<30)) {
2073 process_error = 1;
2074 } else if (is & AHCI_PREG_IS_DHRS(1<<0)) {
2075 u_int32_t tfd;
2076 u_int32_t cmd;
2077 u_int32_t serr;
2078
2079 tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
2080 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
2081 serr = ahci_pread(ap, AHCI_PREG_SERR0x30);
Value stored to 'serr' is never read
2082 if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) &&
2083 (cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) {
2084 DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
2085 " %b, DIAG: %b\n", PORTNAME(ap), tfd,
2086 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2087 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2088 AHCI_PFMT_SERR_DIAG);
2089 process_error = 1;
2090 } else {
2091 /* rfis copy back is in the normal execution path */
2092 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0));
2093 }
2094 }
2095
2096 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
2097 if (process_error) {
2098 u_int32_t tfd, serr;
2099 int err_slot;
2100
2101 tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
2102 serr = ahci_pread(ap, AHCI_PREG_SERR0x30);
2103
2104 if (ap->ap_sactive == 0) {
2105 /* Errored slot is easy to determine from CMD. */
2106 err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,(((ahci_pread(ap, 0x18)) >> 8) & 0x1f)
2107 AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f);
2108
2109 if ((ci_saved & (1 << err_slot)) == 0) {
2110 /*
2111 * Hardware doesn't seem to report correct
2112 * slot number. If there's only one
2113 * outstanding command we can cope,
2114 * otherwise fail all active commands.
2115 */
2116 if (ap->ap_active_cnt == 1)
2117 err_slot = ffs(ap->ap_active) - 1;
2118 else
2119 goto failall;
2120 }
2121
2122 ccb = &ap->ap_ccbs[err_slot];
2123
2124 /* Preserve received taskfile data from the RFIS. */
2125 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis->
rfis), (sizeof(struct ata_fis_d2h)))
2126 sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis->
rfis), (sizeof(struct ata_fis_d2h)))
;
2127 } else
2128 err_slot = -1; /* Must extract error from log page */
2129
2130 DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
2131 " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
2132 AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2133 AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2134 AHCI_PFMT_SERR_DIAG);
2135
2136 /* Turn off ST to clear CI and SACT. */
2137 ahci_port_stop(ap, 0);
2138 need_restart = 1;
2139
2140 /* Clear SERR to enable capturing new errors. */
2141 ahci_pwrite(ap, AHCI_PREG_SERR0x30, serr);
2142
2143 /* Acknowledge the interrupts we can recover from. */
2144 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_TFES(1<<30) |
2145 AHCI_PREG_IS_IFS(1<<27));
2146 is = ahci_pread(ap, AHCI_PREG_IS0x10);
2147
2148 /* If device hasn't cleared its busy status, try to idle it. */
2149 if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)((tfd) & ((1<<7) | (1<<3)))) {
2150
2151 if ((ap->ap_state == AP_S_PMP_PORT_PROBE2) ||
2152 (ap->ap_state == AP_S_ERROR_RECOVERY3)) {
2153 /* can't reset the port here, just make sure
2154 * the operation fails and the port still works.
2155 */
2156 } else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
2157 printf("%s: error on PMP port %d, idling "
2158 "device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2159 ccb->ccb_xa.pmp_port);
2160 if (ahci_pmp_port_softreset(ap,
2161 ccb->ccb_xa.pmp_port) == 0) {
2162 printf("%s: unable to softreset port "
2163 "%d\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2164 ccb->ccb_xa.pmp_port);
2165 if (ahci_pmp_port_portreset(ap,
2166 ccb->ccb_xa.pmp_port)) {
2167 printf("%s: failed to port "
2168 " reset %d, giving up on "
2169 "it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2170 ccb->ccb_xa.pmp_port);
2171 goto fatal;
2172 }
2173 }
2174 } else {
2175 printf("%s: attempting to idle device\n",
2176 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2177 if (ahci_port_softreset(ap)) {
2178 printf("%s: failed to soft reset "
2179 "device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2180 if (ahci_port_portreset(ap, 0)) {
2181 printf("%s: failed to port "
2182 "reset device, give up on "
2183 "it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2184 goto fatal;
2185 }
2186 }
2187 }
2188
2189 /* Had to reset device, can't gather extended info. */
2190 } else if (ap->ap_sactive) {
2191 /* Recover the NCQ error from log page 10h.
2192 * We can only have queued commands active for one port
2193 * at a time, so we know which device errored.
2194 */
2195 ahci_port_read_ncq_error(ap, &err_slot,
2196 ap->ap_pmp_ncq_port);
2197 if (err_slot < 0)
2198 goto failall;
2199
2200 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2201 PORTNAME(ap), err_slot);
2202
2203 ccb = &ap->ap_ccbs[err_slot];
2204 if (ccb->ccb_xa.state != ATA_S_ONCHIP5) {
2205 printf("%s: NCQ errored slot %d is idle"
2206 " (%08x active)\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), err_slot,
2207 ci_saved);
2208 goto failall;
2209 }
2210 } else {
2211 /* Didn't reset, could gather extended info from log. */
2212 }
2213
2214 /*
2215 * If we couldn't determine the errored slot, reset the port
2216 * and fail all the active slots.
2217 */
2218 if (err_slot == -1) {
2219 if (ahci_port_softreset(ap) != 0 &&
2220 ahci_port_portreset(ap, 0) != 0) {
2221 printf("%s: couldn't reset after NCQ error, "
2222 "disabling device.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2223 goto fatal;
2224 }
2225 printf("%s: couldn't recover NCQ error, failing "
2226 "all outstanding commands.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2227 goto failall;
2228 }
2229
2230 /* Clear the failed command in saved CI so completion runs. */
2231 ci_saved &= ~(1 << err_slot);
2232
2233 /* Note the error in the ata_xfer. */
2234 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2234, "ccb->ccb_xa.state == ATA_S_ONCHIP"
))
;
2235 ccb->ccb_xa.state = ATA_S_ERROR3;
2236
2237#ifdef DIAGNOSTIC1
2238 /* There may only be one outstanding standard command now. */
2239 if (ap->ap_sactive == 0) {
2240 tmp = ci_saved;
2241 if (tmp) {
2242 slot = ffs(tmp) - 1;
2243 tmp &= ~(1 << slot);
2244 KASSERT(tmp == 0)((tmp == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 2244, "tmp == 0"))
;
2245 }
2246 }
2247#endif
2248 }
2249
2250 /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts
2251 * when there's a normal SATA device attached.
2252 */
2253 if ((ap->ap_state == AP_S_PMP_PROBE1) &&
2254 (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE(1<<1)) &&
2255 (is & AHCI_PREG_IS_IPMS(1<<23))) {
2256 slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f);
2257 DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
2258 PORTNAME(ap), slot);
2259
2260 ccb = &ap->ap_ccbs[slot];
2261 ccb->ccb_xa.state = ATA_S_ERROR3;
2262
2263 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IPMS(1<<23));
2264 is &= ~AHCI_PREG_IS_IPMS(1<<23);
2265 }
2266
2267 /* ignore IFS errors while resetting a PMP port */
2268 if ((is & AHCI_PREG_IS_IFS(1<<27)) /*&& ap->ap_pmp_ignore_ifs*/) {
2269 DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
2270 "port\n", PORTNAME(ap));
2271
2272 need_restart = 1;
2273 ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
2274 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
2275 is &= ~AHCI_PREG_IS_IFS(1<<27);
2276 goto failall;
2277 }
2278
2279 /* Check for remaining errors - they are fatal. */
2280 if (is & (AHCI_PREG_IS_TFES(1<<30) | AHCI_PREG_IS_HBFS(1<<29) | AHCI_PREG_IS_IFS(1<<27) |
2281 AHCI_PREG_IS_OFS(1<<24) | AHCI_PREG_IS_UFS(1<<4))) {
2282 printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
2283 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), is, AHCI_PFMT_IS"\20" "\040CPDS" "\037TFES" "\036HBFS" "\035HBDS" "\034IFS" "\033INFS"
"\031OFS" "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" "\007PCS"
"\005UFS" "\004SDBS" "\003DSS" "\002PSS" "\001DHRS"
);
2284
2285 /* XXX try recovery first */
2286 goto fatal;
2287 }
2288
2289 /* Fail all outstanding commands if we know the port won't recover. */
2290 if (ap->ap_state == AP_S_FATAL_ERROR4) {
2291fatal:
2292 ap->ap_state = AP_S_FATAL_ERROR4;
2293failall:
2294
2295 /* Ensure port is shut down. */
2296 ahci_port_stop(ap, 1);
2297
2298 /* Error all the active slots. */
2299 ci_masked = ci_saved & *active;
2300 while (ci_masked) {
2301 slot = ffs(ci_masked) - 1;
2302 ccb = &ap->ap_ccbs[slot];
2303 ci_masked &= ~(1 << slot);
2304 ccb->ccb_xa.state = ATA_S_ERROR3;
2305 }
2306
2307 /* Run completion for all active slots. */
2308 ci_saved &= ~*active;
2309
2310 /* Don't restart the port if our problems were deemed fatal. */
2311 if (ap->ap_state == AP_S_FATAL_ERROR4)
2312 need_restart = 0;
2313 }
2314
2315 /*
2316 * CCB completion is detected by noticing its slot's bit in CI has
2317 * changed to zero some time after we activated it.
2318 * If we are polling, we may only be interested in particular slot(s).
2319 */
2320 ci_masked = ~ci_saved & *active & ci_mask;
2321 while (ci_masked) {
2322 slot = ffs(ci_masked) - 1;
2323 ccb = &ap->ap_ccbs[slot];
2324 ci_masked &= ~(1 << slot);
2325
2326 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2327 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2328 " (error)" : "");
2329
2330 bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08)
)
2331 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08)
)
2332 ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08)
)
2333 sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08)
)
;
2334
2335 bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08
))
2336 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08
))
2337 ccb->ccb_slot * sizeof(struct ahci_cmd_table),(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08
))
2338 sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot * sizeof
(struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08
))
;
2339
2340 bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis)
), (0x02))
2341 AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis)
), (0x02))
2342 sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap->
ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis)
), (0x02))
;
2343
2344 *active &= ~(1 << ccb->ccb_slot);
2345 /* Copy the rfis into the ccb if we were asked for it */
2346 if (ccb->ccb_xa.state == ATA_S_ONCHIP5 &&
2347 ccb->ccb_xa.flags & ATA_F_GET_RFIS(1<<8)) {
2348 memcpy(&ccb->ccb_xa.rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis->
rfis), (sizeof(struct ata_fis_d2h)))
2349 ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis->
rfis), (sizeof(struct ata_fis_d2h)))
2350 sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis->
rfis), (sizeof(struct ata_fis_d2h)))
;
2351 }
2352
2353 ccb->ccb_done(ccb);
2354
2355 processed |= 1 << ccb->ccb_slot;
2356 }
2357
2358 if (need_restart) {
2359 /* Restart command DMA on the port */
2360 ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
2361
2362 /* Re-enable outstanding commands on port. */
2363 if (ci_saved) {
2364#ifdef DIAGNOSTIC1
2365 tmp = ci_saved;
2366 while (tmp) {
2367 slot = ffs(tmp) - 1;
2368 tmp &= ~(1 << slot);
2369 ccb = &ap->ap_ccbs[slot];
2370 KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2370, "ccb->ccb_xa.state == ATA_S_ONCHIP"
))
;
2371 KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap->
ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)"
))
2372 (!!ap->ap_sactive))(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap->
ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)"
))
;
2373 }
2374#endif
2375 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2376 "re-enabling%s slots %08x\n", PORTNAME(ap),
2377 ap->ap_sactive ? " NCQ" : "", ci_saved);
2378
2379 if (ap->ap_sactive)
2380 ahci_pwrite(ap, AHCI_PREG_SACT0x34, ci_saved);
2381 ahci_pwrite(ap, AHCI_PREG_CI0x38, ci_saved);
2382 }
2383 }
2384
2385 return (processed);
2386}
2387
2388struct ahci_ccb *
2389ahci_get_ccb(struct ahci_port *ap)
2390{
2391 struct ahci_ccb *ccb;
2392
2393 mtx_enter(&ap->ap_ccb_mtx);
2394 ccb = TAILQ_FIRST(&ap->ap_ccb_free)((&ap->ap_ccb_free)->tqh_first);
2395 if (ccb != NULL((void *)0)) {
2396 KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2396, "ccb->ccb_xa.state == ATA_S_PUT"
))
;
2397 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb)
->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry
.tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb
)->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = (ccb
)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((
void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1); } while
(0)
;
2398 ccb->ccb_xa.state = ATA_S_SETUP0;
2399 }
2400 mtx_leave(&ap->ap_ccb_mtx);
2401
2402 return (ccb);
2403}
2404
2405void
2406ahci_put_ccb(struct ahci_ccb *ccb)
2407{
2408 struct ahci_port *ap = ccb->ccb_port;
2409
2410#ifdef DIAGNOSTIC1
2411 if (ccb->ccb_xa.state != ATA_S_COMPLETE2 &&
2412 ccb->ccb_xa.state != ATA_S_TIMEOUT4 &&
2413 ccb->ccb_xa.state != ATA_S_ERROR3) {
2414 printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2415 "slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state,
2416 ccb->ccb_slot);
2417 }
2418#endif
2419
2420 ccb->ccb_xa.state = ATA_S_PUT6;
2421 mtx_enter(&ap->ap_ccb_mtx);
2422 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry
.tqe_prev = (&ap->ap_ccb_free)->tqh_last; *(&ap
->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free
)->tqh_last = &(ccb)->ccb_entry.tqe_next; } while (
0)
;
2423 mtx_leave(&ap->ap_ccb_mtx);
2424}
2425
2426struct ahci_ccb *
2427ahci_get_err_ccb(struct ahci_port *ap)
2428{
2429 struct ahci_ccb *err_ccb;
2430 u_int32_t sact;
2431
2432 splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__
); } } while (0)
;
2433
2434 /* No commands may be active on the chip. */
2435 sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2436 if (sact != 0)
2437 printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2438 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2438, "ahci_pread(ap, AHCI_PREG_CI) == 0"
))
;
2439
2440#ifdef DIAGNOSTIC1
2441 KASSERT(ap->ap_err_busy == 0)((ap->ap_err_busy == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2441, "ap->ap_err_busy == 0"
))
;
2442 ap->ap_err_busy = 1;
2443#endif
2444 /* Save outstanding command state. */
2445 ap->ap_err_saved_active = ap->ap_active;
2446 ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2447 ap->ap_err_saved_sactive = ap->ap_sactive;
2448
2449 /*
2450 * Pretend we have no commands outstanding, so that completions won't
2451 * run prematurely.
2452 */
2453 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2454
2455 /*
2456 * Grab a CCB to use for error recovery. This should never fail, as
2457 * we ask atascsi to reserve one for us at init time.
2458 */
2459 err_ccb = ap->ap_ccb_err;
2460 err_ccb->ccb_xa.flags = 0;
2461 err_ccb->ccb_xa.state = ATA_S_SETUP0;
2462 err_ccb->ccb_done = ahci_empty_done;
2463
2464 return (err_ccb);
2465}
2466
2467void
2468ahci_put_err_ccb(struct ahci_ccb *ccb)
2469{
2470 struct ahci_port *ap = ccb->ccb_port;
2471 u_int32_t sact;
2472
2473 splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__
); } } while (0)
;
2474
2475#ifdef DIAGNOSTIC1
2476 KASSERT(ap->ap_err_busy)((ap->ap_err_busy) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c"
, 2476, "ap->ap_err_busy"))
;
2477#endif
2478 /* No commands may be active on the chip */
2479 sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2480 if (sact != 0)
2481 printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact);
2482 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2482, "ahci_pread(ap, AHCI_PREG_CI) == 0"
))
;
2483
2484 /* Done with the CCB */
2485 KASSERT(ccb == ap->ap_ccb_err)((ccb == ap->ap_ccb_err) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2485, "ccb == ap->ap_ccb_err"
))
;
2486
2487 /* Restore outstanding command state */
2488 ap->ap_sactive = ap->ap_err_saved_sactive;
2489 ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2490 ap->ap_active = ap->ap_err_saved_active;
2491
2492#ifdef DIAGNOSTIC1
2493 ap->ap_err_busy = 0;
2494#endif
2495}
2496
2497struct ahci_ccb *
2498ahci_get_pmp_ccb(struct ahci_port *ap)
2499{
2500 struct ahci_ccb *ccb;
2501 u_int32_t sact;
2502
2503 /* some PMP commands need to be issued on slot 1,
2504 * particularly the command that clears SRST and
2505 * fetches the device signature.
2506 *
2507 * ensure the chip is idle and ccb 1 is available.
2508 */
2509 splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__
); } } while (0)
;
2510
2511 sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2512 if (sact != 0)
2513 printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
2514 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2514, "ahci_pread(ap, AHCI_PREG_CI) == 0"
))
;
2515
2516 ccb = &ap->ap_ccbs[1];
2517 KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2517, "ccb->ccb_xa.state == ATA_S_PUT"
))
;
2518 ccb->ccb_xa.flags = 0;
2519 ccb->ccb_done = ahci_pmp_cmd_done;
2520
2521 mtx_enter(&ap->ap_ccb_mtx);
2522 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb)
->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry
.tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb
)->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = (ccb
)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((
void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1); } while
(0)
;
2523 mtx_leave(&ap->ap_ccb_mtx);
2524
2525 return ccb;
2526}
2527
2528void
2529ahci_put_pmp_ccb(struct ahci_ccb *ccb)
2530{
2531 struct ahci_port *ap = ccb->ccb_port;
2532 u_int32_t sact;
2533
2534 /* make sure this is the right ccb */
2535 KASSERT(ccb == &ap->ap_ccbs[1])((ccb == &ap->ap_ccbs[1]) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2535, "ccb == &ap->ap_ccbs[1]"
))
;
2536
2537 /* No commands may be active on the chip */
2538 sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2539 if (sact != 0)
2540 printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact);
2541 KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2541, "ahci_pread(ap, AHCI_PREG_CI) == 0"
))
;
2542
2543 ccb->ccb_xa.state = ATA_S_PUT6;
2544 mtx_enter(&ap->ap_ccb_mtx);
2545 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void *)0); (ccb)->ccb_entry
.tqe_prev = (&ap->ap_ccb_free)->tqh_last; *(&ap
->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free
)->tqh_last = &(ccb)->ccb_entry.tqe_next; } while (
0)
;
2546 mtx_leave(&ap->ap_ccb_mtx);
2547}
2548
2549int
2550ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port)
2551{
2552 struct ahci_ccb *ccb;
2553 struct ahci_cmd_hdr *cmd_slot;
2554 u_int32_t cmd;
2555 struct ata_fis_h2d *fis;
2556 int rc = EIO5, oldstate;
2557
2558 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2559 oldstate = ap->ap_state;
2560 ap->ap_state = AP_S_ERROR_RECOVERY3;
2561
2562 /* Save command register state. */
2563 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
2564
2565 /* Port should have been idled already. Start it. */
2566 KASSERT((cmd & AHCI_PREG_CMD_CR) == 0)(((cmd & (1<<15)) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/ic/ahci.c", 2566, "(cmd & AHCI_PREG_CMD_CR) == 0"
))
;
2567 ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
2568
2569 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2570 ccb = ahci_get_err_ccb(ap);
2571 ccb->ccb_xa.flags = ATA_F_NOWAIT(1<<2) | ATA_F_READ(1<<0) | ATA_F_POLL(1<<3);
2572 ccb->ccb_xa.data = ap->ap_err_scratch;
2573 ccb->ccb_xa.datalen = 512;
2574 cmd_slot = ccb->ccb_cmd_hdr;
2575 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
2576
2577 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2578 fis->type = ATA_FIS_TYPE_H2D0x27;
2579 fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | pmp_port;
2580 fis->command = ATA_C_READ_LOG_EXT0x2f;
2581 fis->lba_low = 0x10; /* queued error log page (10h) */
2582 fis->sector_count = 1; /* number of sectors (1) */
2583 fis->sector_count_exp = 0;
2584 fis->lba_mid = 0; /* starting offset */
2585 fis->lba_mid_exp = 0;
2586 fis->device = 0;
2587
2588 htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(pmp_port << 12))))
2589 (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 |
(pmp_port << 12))))
;
2590
2591 if (ahci_load_prdt(ccb) != 0) {
2592 rc = ENOMEM12; /* XXX caller must abort all commands */
2593 goto err;
2594 }
2595
2596 ccb->ccb_xa.state = ATA_S_PENDING1;
2597 if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0 ||
2598 ccb->ccb_xa.state == ATA_S_ERROR3)
2599 goto err;
2600
2601 rc = 0;
2602err:
2603 /* Abort our command, if it failed, by stopping command DMA. */
2604 if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) {
2605 printf("%s: log page read failed, slot %d was still active.\n",
2606 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot);
2607 ahci_port_stop(ap, 0);
2608 }
2609
2610 /* Done with the error CCB now. */
2611 ahci_unload_prdt(ccb);
2612 ahci_put_err_ccb(ccb);
2613
2614 /* Extract failed register set and tags from the scratch space. */
2615 if (rc == 0) {
2616 struct ata_log_page_10h *log;
2617 int err_slot;
2618
2619 log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2620 if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)((log->err_regs.type) & (0x80))) {
2621 /* Not queued bit was set - wasn't an NCQ error? */
2622 printf("%s: read NCQ error page, but not an NCQ "
2623 "error?\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2624 rc = ESRCH3;
2625 } else {
2626 /* Copy back the log record as a D2H register FIS. */
2627 *err_slotp = err_slot = log->err_regs.type &
2628 ATA_LOG_10H_TYPE_TAG_MASK0x1f;
2629
2630 ccb = &ap->ap_ccbs[err_slot];
2631 memcpy(&ccb->ccb_xa.rfis, &log->err_regs,__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs
), (sizeof(struct ata_fis_d2h)))
2632 sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs
), (sizeof(struct ata_fis_d2h)))
;
2633 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H0x34;
2634 ccb->ccb_xa.rfis.flags = 0;
2635 }
2636 }
2637
2638 /* Restore saved CMD register state */
2639 ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
2640 ap->ap_state = oldstate;
2641
2642 return (rc);
2643}
2644
2645struct ahci_dmamem *
2646ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
2647{
2648 struct ahci_dmamem *adm;
2649 int nsegs;
2650
2651 adm = malloc(sizeof(*adm), M_DEVBUF2, M_NOWAIT0x0002 | M_ZERO0x0008);
2652 if (adm == NULL((void *)0))
2653 return (NULL((void *)0));
2654
2655 adm->adm_size = size;
2656
2657 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size
), (1), (size), (0), (0x0001 | 0x0002), (&adm->adm_map
))
2658 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map)(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size
), (1), (size), (0), (0x0001 | 0x0002), (&adm->adm_map
))
!= 0)
2659 goto admfree;
2660
2661 if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size
), ((1 << 12)), (0), (&adm->adm_seg), (1), (&
nsegs), (0x0001 | 0x1000))
2662 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO)(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size
), ((1 << 12)), (0), (&adm->adm_seg), (1), (&
nsegs), (0x0001 | 0x1000))
!= 0)
2663 goto destroy;
2664
2665 if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm
->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001
| 0x0004))
2666 &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT)(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm
->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001
| 0x0004))
!= 0)
2667 goto free;
2668
2669 if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm->
adm_map), (adm->adm_kva), (size), (((void *)0)), (0x0001))
2670 NULL, BUS_DMA_NOWAIT)(*(sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm->
adm_map), (adm->adm_kva), (size), (((void *)0)), (0x0001))
!= 0)
2671 goto unmap;
2672
2673 return (adm);
2674
2675unmap:
2676 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm->
adm_kva), (size))
;
2677free:
2678 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (&
adm->adm_seg), (1))
;
2679destroy:
2680 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm
->adm_map))
;
2681admfree:
2682 free(adm, M_DEVBUF2, sizeof(*adm));
2683
2684 return (NULL((void *)0));
2685}
2686
2687void
2688ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
2689{
2690 bus_dmamap_unload(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (adm
->adm_map))
;
2691 bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm->
adm_kva), (adm->adm_size))
;
2692 bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (&
adm->adm_seg), (1))
;
2693 bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm
->adm_map))
;
2694 free(adm, M_DEVBUF2, sizeof(*adm));
2695}
2696
2697u_int32_t
2698ahci_read(struct ahci_softc *sc, bus_size_t r)
2699{
2700 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2701 BUS_SPACE_BARRIER_READ0x01);
2702 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)((sc->sc_iot)->read_4((sc->sc_ioh), (r))));
2703}
2704
2705void
2706ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2707{
2708 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v)((sc->sc_iot)->write_4((sc->sc_ioh), (r), (v)));
2709 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2710 BUS_SPACE_BARRIER_WRITE0x02);
2711}
2712
2713int
2714ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
2715 u_int32_t target)
2716{
2717 int i;
2718
2719 for (i = 0; i < 1000; i++) {
2720 if ((ahci_read(sc, r) & mask) != target)
2721 return (0);
2722 delay(1000)(*delay_func)(1000);
2723 }
2724
2725 return (1);
2726}
2727
2728u_int32_t
2729ahci_pread(struct ahci_port *ap, bus_size_t r)
2730{
2731 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2732 BUS_SPACE_BARRIER_READ0x01);
2733 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)((ap->ap_sc->sc_iot)->read_4((ap->ap_ioh), (r))));
2734}
2735
2736void
2737ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2738{
2739 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v)((ap->ap_sc->sc_iot)->write_4((ap->ap_ioh), (r), (
v)))
;
2740 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2741 BUS_SPACE_BARRIER_WRITE0x02);
2742}
2743
2744int
2745ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
2746 u_int32_t target, int n)
2747{
2748 int i;
2749
2750 for (i = 0; i < n * 1000; i++) {
2751 if ((ahci_pread(ap, r) & mask) == target)
2752 return (0);
2753 delay(1000)(*delay_func)(1000);
2754 }
2755
2756 return (1);
2757}
2758
2759int
2760ahci_ata_probe(void *xsc, int port, int lun)
2761{
2762 struct ahci_softc *sc = xsc;
2763 struct ahci_port *ap = sc->sc_ports[port];
2764
2765 if (ap == NULL((void *)0))
2766 return (ATA_PORT_T_NONE0);
2767
2768 if (lun != 0) {
2769 int pmp_port = lun - 1;
2770 if (pmp_port >= ap->ap_pmp_ports) {
2771 return (ATA_PORT_T_NONE0);
2772 }
2773 return (ahci_pmp_port_probe(ap, pmp_port));
2774 } else {
2775 return (ahci_port_signature(ap));
2776 }
2777}
2778
2779void
2780ahci_ata_free(void *xsc, int port, int lun)
2781{
2782
2783}
2784
2785struct ata_xfer *
2786ahci_ata_get_xfer(void *aaa_cookie, int port)
2787{
2788 struct ahci_softc *sc = aaa_cookie;
2789 struct ahci_port *ap = sc->sc_ports[port];
2790 struct ahci_ccb *ccb;
2791
2792 ccb = ahci_get_ccb(ap);
2793 if (ccb == NULL((void *)0)) {
2794 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
2795 PORTNAME(ap));
2796 return (NULL((void *)0));
2797 }
2798
2799 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
2800 PORTNAME(ap), ccb->ccb_slot);
2801
2802 return ((struct ata_xfer *)ccb);
2803}
2804
2805void
2806ahci_ata_put_xfer(struct ata_xfer *xa)
2807{
2808 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2809
2810 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
2811
2812 ahci_put_ccb(ccb);
2813}
2814
2815void
2816ahci_ata_cmd(struct ata_xfer *xa)
2817{
2818 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2819 struct ahci_cmd_hdr *cmd_slot;
2820 int s;
2821 u_int16_t flags;
2822
2823 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR4)
2824 goto failcmd;
2825
2826 ccb->ccb_done = ahci_ata_cmd_done;
2827
2828 cmd_slot = ccb->ccb_cmd_hdr;
2829 flags = 5 /* FIS length (in DWORDs) */;
2830 flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12;
2831
2832 if (xa->flags & ATA_F_WRITE(1<<1))
2833 flags |= AHCI_CMD_LIST_FLAG_W(1<<6);
2834
2835 if (xa->flags & ATA_F_PACKET(1<<5))
2836 flags |= AHCI_CMD_LIST_FLAG_A(1<<5);
2837
2838 htolem16(&cmd_slot->flags, flags)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(flags
)))
;
2839
2840 if (ahci_load_prdt(ccb) != 0)
2841 goto failcmd;
2842
2843 timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
2844
2845 xa->state = ATA_S_PENDING1;
2846
2847 if (xa->flags & ATA_F_POLL(1<<3))
2848 ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
2849 else {
2850 s = splbio()splraise(0x3);
2851 timeout_add_msec(&xa->stimeout, xa->timeout);
2852 ahci_start(ccb);
2853 splx(s)spllower(s);
2854 }
2855
2856 return;
2857
2858failcmd:
2859 s = splbio()splraise(0x3);
2860 xa->state = ATA_S_ERROR3;
2861 ata_complete(xa);
2862 splx(s)spllower(s);
2863}
2864
2865void
2866ahci_pmp_cmd_done(struct ahci_ccb *ccb)
2867{
2868 struct ata_xfer *xa = &ccb->ccb_xa;
2869
2870 if (xa->state == ATA_S_ONCHIP5 || xa->state == ATA_S_ERROR3)
2871 ahci_issue_pending_commands(ccb->ccb_port,
2872 xa->flags & ATA_F_NCQ(1<<6));
2873
2874 xa->state = ATA_S_COMPLETE2;
2875}
2876
2877
2878void
2879ahci_ata_cmd_done(struct ahci_ccb *ccb)
2880{
2881 struct ata_xfer *xa = &ccb->ccb_xa;
2882
2883 timeout_del(&xa->stimeout);
2884
2885 if (xa->state == ATA_S_ONCHIP5 || xa->state == ATA_S_ERROR3)
2886 ahci_issue_pending_commands(ccb->ccb_port,
2887 xa->flags & ATA_F_NCQ(1<<6));
2888
2889 ahci_unload_prdt(ccb);
2890
2891 if (xa->state == ATA_S_ONCHIP5)
2892 xa->state = ATA_S_COMPLETE2;
2893#ifdef DIAGNOSTIC1
2894 else if (xa->state != ATA_S_ERROR3 && xa->state != ATA_S_TIMEOUT4)
2895 printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
2896 "slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), xa->state,
2897 ccb->ccb_slot);
2898#endif
2899 if (xa->state != ATA_S_TIMEOUT4)
2900 ata_complete(xa);
2901}
2902
2903void
2904ahci_ata_cmd_timeout(void *arg)
2905{
2906 struct ahci_ccb *ccb = arg;
2907 struct ata_xfer *xa = &ccb->ccb_xa;
2908 struct ahci_port *ap = ccb->ccb_port;
2909 int s, ccb_was_started, ncq_cmd;
2910 volatile u_int32_t *active;
2911
2912 s = splbio()splraise(0x3);
2913
2914 ncq_cmd = (xa->flags & ATA_F_NCQ(1<<6));
2915 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
2916
2917 if (ccb->ccb_xa.state == ATA_S_PENDING1) {
2918 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
2919 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
2920 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void *)0)) (ccb)
->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry
.tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = (
ccb)->ccb_entry.tqe_prev; *(ccb)->ccb_entry.tqe_prev = (
ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) =
((void *)-1); ((ccb)->ccb_entry.tqe_next) = ((void *)-1);
} while (0)
;
2921 ccb_was_started = 0;
2922 } else if (ccb->ccb_xa.state == ATA_S_ONCHIP5 && ahci_port_intr(ap,
2923 1 << ccb->ccb_slot)) {
2924 DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
2925 "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
2926 goto ret;
2927 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP5) {
2928 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
2929 "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
2930 ISSET(*active, 1 << ccb->ccb_slot) ?
2931 " but slot is still active?" : ".");
2932 goto ret;
2933 } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb
->ccb_slot))
2934 AHCI_PREG_CI), 1 << ccb->ccb_slot)((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb
->ccb_slot))
&& ISSET(*active,((*active) & (1 << ccb->ccb_slot))
2935 1 << ccb->ccb_slot)((*active) & (1 << ccb->ccb_slot))) {
2936 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
2937 "IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
2938 ccb->ccb_slot);
2939 *active &= ~(1 << ccb->ccb_slot);
2940 ccb->ccb_done(ccb);
2941 goto ret;
2942 } else {
2943 ccb_was_started = 1;
2944 }
2945
2946 /* Complete the slot with a timeout error. */
2947 ccb->ccb_xa.state = ATA_S_TIMEOUT4;
2948 *active &= ~(1 << ccb->ccb_slot);
2949 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
2950 ccb->ccb_done(ccb); /* This won't issue pending commands or run the
2951 atascsi completion. */
2952
2953 /* Reset port to abort running command. */
2954 if (ccb_was_started) {
2955 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
2956 "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
2957 ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
2958 if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
2959 != 0) {
2960 printf("%s: failed to reset port during timeout "
2961 "handling, disabling it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2962 ap->ap_state = AP_S_FATAL_ERROR4;
2963 }
2964
2965 /* Restart any other commands that were aborted by the reset. */
2966 if (*active) {
2967 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
2968 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
2969 *active);
2970 if (ncq_cmd)
2971 ahci_pwrite(ap, AHCI_PREG_SACT0x34, *active);
2972 ahci_pwrite(ap, AHCI_PREG_CI0x38, *active);
2973 }
2974 }
2975
2976 /* Issue any pending commands now. */
2977 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
2978 if (ccb_was_started)
2979 ahci_issue_pending_commands(ap, ncq_cmd);
2980 else if (ap->ap_active == 0)
2981 ahci_issue_pending_ncq_commands(ap);
2982
2983 /* Complete the timed out ata_xfer I/O (may generate new I/O). */
2984 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
2985 ata_complete(xa);
2986
2987 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
2988ret:
2989 splx(s)spllower(s);
2990}
2991
2992void
2993ahci_empty_done(struct ahci_ccb *ccb)
2994{
2995 if (ccb->ccb_xa.state != ATA_S_ERROR3)
2996 ccb->ccb_xa.state = ATA_S_COMPLETE2;
2997}
2998
2999int
3000ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap)
3001{
3002 struct ahci_ccb *ccb;
3003 struct ata_fis_h2d *fis;
3004 int error;
3005
3006 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3007 ccb->ccb_xa.flags = ATA_F_POLL(1<<3) | ATA_F_GET_RFIS(1<<8);
3008 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f;
3009 ccb->ccb_xa.state = ATA_S_PENDING1;
3010
3011 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
3012 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3013 fis->type = ATA_FIS_TYPE_H2D0x27;
3014 fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | SATA_PMP_CONTROL_PORT0x0f;
3015 fis->command = ATA_C_READ_PM0xe4;
3016 fis->features = which;
3017 fis->device = target | ATA_H2D_DEVICE_LBA0x40;
3018 fis->control = ATA_FIS_CONTROL_4BIT0x08;
3019
3020 if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
3021 error = 1;
3022 } else {
3023 *datap = ccb->ccb_xa.rfis.sector_count |
3024 (ccb->ccb_xa.rfis.lba_low << 8) |
3025 (ccb->ccb_xa.rfis.lba_mid << 16) |
3026 (ccb->ccb_xa.rfis.lba_high << 24);
3027 error = 0;
3028 }
3029 ahci_put_pmp_ccb(ccb);
3030 return (error);
3031}
3032
3033int
3034ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
3035{
3036 struct ahci_ccb *ccb;
3037 struct ata_fis_h2d *fis;
3038 int error;
3039
3040 ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3041 ccb->ccb_xa.flags = ATA_F_POLL(1<<3);
3042 ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f;
3043 ccb->ccb_xa.state = ATA_S_PENDING1;
3044
3045 memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct
ahci_cmd_table)))
;
3046 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3047 fis->type = ATA_FIS_TYPE_H2D0x27;
3048 fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | SATA_PMP_CONTROL_PORT0x0f;
3049 fis->command = ATA_C_WRITE_PM0xe8;
3050 fis->features = which;
3051 fis->device = target | ATA_H2D_DEVICE_LBA0x40;
3052 fis->sector_count = (u_int8_t)data;
3053 fis->lba_low = (u_int8_t)(data >> 8);
3054 fis->lba_mid = (u_int8_t)(data >> 16);
3055 fis->lba_high = (u_int8_t)(data >> 24);
3056 fis->control = ATA_FIS_CONTROL_4BIT0x08;
3057
3058 error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
3059 ahci_put_pmp_ccb(ccb);
3060 return (error);
3061}
3062
3063int
3064ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap)
3065{
3066 int error;
3067
3068 error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS0, datap);
3069 if (error == 0)
3070 error = ahci_pmp_write(ap, target, SATA_PMREG_SERR1, -1);
3071 if (error)
3072 *datap = 0;
3073
3074 return (error);
3075}
3076
3077int
3078ahci_pmp_identify(struct ahci_port *ap, int *ret_nports)
3079{
3080 u_int32_t chipid;
3081 u_int32_t rev;
3082 u_int32_t nports;
3083 u_int32_t features;
3084 u_int32_t enabled;
3085 int s;
3086
3087 s = splbio()splraise(0x3);
3088
3089 if (ahci_pmp_read(ap, 15, 0, &chipid) ||
3090 ahci_pmp_read(ap, 15, 1, &rev) ||
3091 ahci_pmp_read(ap, 15, 2, &nports) ||
3092 ahci_pmp_read(ap, 15, SATA_PMREG_FEA64, &features) ||
3093 ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN96, &enabled)) {
3094 printf("%s: port multiplier identification failed\n",
3095 PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
3096 splx(s)spllower(s);
3097 return (1);
3098 }
3099 splx(s)spllower(s);
3100
3101 nports &= 0x0F;
3102
3103 /* ignore SEMB port on SiI3726 port multiplier chips */
3104 if (chipid == 0x37261095) {
3105 nports--;
3106 }
3107
3108 printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
3109 "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), chipid, rev,
3110 SATA_PFMT_PM_REV"\20" "\003PM1.1" "\002PM1.0", nports, features, SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST", enabled,
3111 SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST");
3112
3113 *ret_nports = nports;
3114 return (0);
3115}
3116
3117
3118#ifdef HIBERNATE1
3119void
3120ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb)
3121{
3122 ccb->ccb_cmd_hdr->prdbc = 0;
3123 ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
3124}
3125
3126int
3127ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb)
3128{
3129 u_int32_t is, ci_saved;
3130 int process_error = 0;
3131
3132 is = ahci_pread(ap, AHCI_PREG_IS0x10);
3133
3134 ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38);
3135
3136 if (is & AHCI_PREG_IS_DHRS(1<<0)) {
3137 u_int32_t tfd;
3138 u_int32_t cmd;
3139
3140 tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
3141 cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
3142 if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) &&
3143 (cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) {
3144 process_error = 1;
3145 } else {
3146 ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0));
3147 }
3148 } else if (is & (AHCI_PREG_IS_TFES(1<<30) | AHCI_PREG_IS_HBFS(1<<29) |
3149 AHCI_PREG_IS_IFS(1<<27) | AHCI_PREG_IS_OFS(1<<24) | AHCI_PREG_IS_UFS(1<<4))) {
3150 process_error = 1;
3151 }
3152
3153 /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
3154 if (process_error) {
3155
3156 /* Turn off ST to clear CI and SACT. */
3157 ahci_port_stop(ap, 0);
3158
3159 /* just return an error indicator? we can't meaningfully
3160 * recover, and on the way back out we'll DVACT_RESUME which
3161 * resets and reinits the port.
3162 */
3163 return (EIO5);
3164 }
3165
3166 /* command is finished when the bit in CI for the slot goes to 0 */
3167 if (ci_saved & (1 << ccb->ccb_slot)) {
3168 return (EAGAIN35);
3169 }
3170
3171 return (0);
3172}
3173
3174void
3175ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
3176{
3177 struct ata_xfer *xa = &ccb->ccb_xa;
3178 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
3179 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
3180 int i;
3181 paddr_t data_phys;
3182 u_int64_t data_bus_phys;
3183 vaddr_t data_addr;
3184 size_t seglen;
3185 size_t buflen;
3186
3187 if (xa->datalen == 0) {
3188 ccb->ccb_cmd_hdr->prdtl = 0;
3189 return;
3190 }
3191
3192 /* derived from i386/amd64 _bus_dma_load_buffer;
3193 * for amd64 the buffer will always be dma safe.
3194 */
3195
3196 buflen = xa->datalen;
3197 data_addr = (vaddr_t)xa->data;
3198 for (i = 0; buflen > 0; i++) {
3199 pmap_extract(pmap_kernel()(&kernel_pmap_store), data_addr, &data_phys);
3200 data_bus_phys = data_phys;
3201
3202 seglen = PAGE_SIZE(1 << 12) - ((u_long)data_addr & PGOFSET((1 << 12) - 1));
3203 if (buflen < seglen)
3204 seglen = buflen;
3205
3206 ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
3207
3208 data_addr += seglen;
3209 buflen -= seglen;
3210 }
3211
3212 htolem16(&cmd_slot->prdtl, i)(*(__uint16_t *)(&cmd_slot->prdtl) = ((__uint16_t)(i))
)
;
3213}
3214
3215int
3216ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
3217 int op, void *page)
3218{
3219 /* we use the 'real' ahci_port and ahci_softc here, but
3220 * never write to them
3221 */
3222 struct {
3223 struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */
3224 struct ahci_rfis rfis; /* 1k aligned, 256 bytes */
3225 /* cmd table isn't actually used because of mysteries */
3226 struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */
3227 struct ahci_port *ap;
3228 struct ahci_ccb ccb_buf;
3229 struct ahci_ccb *ccb;
3230 struct ahci_cmd_hdr *hdr_buf;
3231 int pmp_port;
3232 daddr_t poffset;
3233 size_t psize;
3234 } *my = page;
3235 struct ata_fis_h2d *fis;
3236 u_int32_t sector_count;
3237 struct ahci_cmd_hdr *cmd_slot;
3238 int rc;
3239 int timeout;
3240 u_int16_t flags;
3241
3242 if (op == HIB_INIT-1) {
3243 struct device *disk;
3244 struct device *scsibus;
3245 struct ahci_softc *sc;
3246 extern struct cfdriver sd_cd;
3247 struct scsi_link *link;
3248 struct scsibus_softc *bus_sc;
3249 int port;
3250 paddr_t page_phys;
3251 u_int64_t item_phys;
3252 u_int32_t cmd;
3253
3254 my->poffset = blkno;
3255 my->psize = size;
3256
3257 /* map dev to an ahci port */
3258 disk = disk_lookup(&sd_cd, DISKUNIT(dev)(((unsigned)((dev) & 0xff) | (((dev) & 0xffff0000) >>
8)) / 16)
);
3259 scsibus = disk->dv_parent;
3260 sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
3261
3262 /* find the scsi_link for the device, which has the port */
3263 port = -1;
3264 bus_sc = (struct scsibus_softc *)scsibus;
3265 SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list)for((link) = ((&bus_sc->sc_link_list)->slh_first); (
link) != ((void *)0); (link) = ((link)->bus_list.sle_next)
)
{
3266 if (link->device_softc == disk) {
3267 port = link->target;
3268 if (link->lun > 0)
3269 my->pmp_port = link->lun - 1;
3270 else
3271 my->pmp_port = 0;
3272
3273 break;
3274 }
3275 }
3276 if (port == -1) {
3277 /* don't know where the disk is */
3278 return (EIO5);
3279 }
3280
3281 my->ap = sc->sc_ports[port];
3282
3283 /* we're going to use the first command slot,
3284 * so ensure it's not already in use
3285 */
3286 if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT6) {
3287 /* this shouldn't happen, we should be idle */
3288 return (EIO5);
3289 }
3290
3291 /* stop the port so we can relocate to the hibernate page */
3292 if (ahci_port_stop(my->ap, 1)) {
3293 return (EIO5);
3294 }
3295 ahci_pwrite(my->ap, AHCI_PREG_SCTL0x2c, 0);
3296
3297 pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)page, &page_phys);
3298
3299 /* Setup RFIS base address */
3300 item_phys = page_phys + ((void *)&my->rfis - page);
3301 ahci_pwrite(my->ap, AHCI_PREG_FBU0x0c,
3302 (u_int32_t)(item_phys >> 32));
3303 ahci_pwrite(my->ap, AHCI_PREG_FB0x08, (u_int32_t)item_phys);
3304
3305 /* Enable FIS reception and activate port. */
3306 cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
3307 cmd |= AHCI_PREG_CMD_FRE(1<<4) | AHCI_PREG_CMD_POD(1<<2) |
3308 AHCI_PREG_CMD_SUD(1<<1);
3309 ahci_pwrite(my->ap, AHCI_PREG_CMD0x18, cmd |
3310 AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
3311
3312 /* Check whether port activated. */
3313 cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
3314 if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
3315 return (EIO5);
3316 }
3317
3318 /* Set up the single CCB */
3319 my->ccb = &my->ccb_buf;
3320 my->ccb->ccb_slot = 0;
3321 my->ccb->ccb_port = my->ap;
3322
3323 /* Setup command list base address */
3324 item_phys = page_phys + ((void *)&my->cmd_hdr - page);
3325 ahci_pwrite(my->ap, AHCI_PREG_CLBU0x04,
3326 (u_int32_t)(item_phys >> 32));
3327 ahci_pwrite(my->ap, AHCI_PREG_CLB0x00, (u_int32_t)item_phys);
3328
3329 my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
3330
3331 /* use existing cmd table - moving to a new one fails */
3332 my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
3333 pmap_extract(pmap_kernel()(&kernel_pmap_store),
3334 (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table)((void *)(my->ap->ap_dmamem_cmd_table)->adm_kva),
3335 &page_phys);
3336 item_phys = page_phys;
3337#if 0
3338 /* use cmd table in hibernate page (doesn't work) */
3339 my->ccb->ccb_cmd_table = &my->cmd_table;
3340 item_phys = page_phys + ((void *)&my->cmd_table - page);
3341#endif
3342 htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys)(*(__uint64_t *)(&my->ccb->ccb_cmd_hdr->ctba) = (
(__uint64_t)(item_phys)))
;
3343
3344 my->ccb->ccb_xa.fis =
3345 (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
3346 my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
3347 my->ccb->ccb_xa.tag = 0;
3348
3349 /* Wait for ICC change to complete */
3350 ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((my->ap), (0x18), (0xf0000000), 0, (1));
3351
3352 if (ahci_port_start(my->ap, 0)((my->ap)->ap_sc->sc_port_start((my->ap), (0)))) {
3353 return (EIO5);
3354 }
3355
3356 /* Flush interrupts for port */
3357 ahci_pwrite(my->ap, AHCI_PREG_IS0x10, ahci_pread(my->ap,
3358 AHCI_PREG_IS0x10));
3359 ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
3360
3361 ahci_enable_interrupts(my->ap);
3362 return (0);
3363 } else if (op == HIB_DONE-2) {
3364 ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME4);
3365 return (0);
3366 }
3367
3368 if (blkno > my->psize)
3369 return (E2BIG7);
3370 blkno += my->poffset;
3371
3372 /* build fis */
3373 sector_count = size / 512; /* dlg promises this is okay */
3374 my->ccb->ccb_xa.flags = op == HIB_W1 ? ATA_F_WRITE(1<<1) : ATA_F_READ(1<<0);
3375 fis = my->ccb->ccb_xa.fis;
3376 fis->flags = ATA_H2D_FLAGS_CMD(1<<7) | my->pmp_port;
3377 fis->lba_low = blkno & 0xff;
3378 fis->lba_mid = (blkno >> 8) & 0xff;
3379 fis->lba_high = (blkno >> 16) & 0xff;
3380
3381 if (sector_count > 0x100 || blkno > 0xfffffff) {
3382 /* Use LBA48 */
3383 fis->command = op == HIB_W1 ? ATA_C_WRITEDMA_EXT0x35 :
3384 ATA_C_READDMA_EXT0x25;
3385 fis->device = ATA_H2D_DEVICE_LBA0x40;
3386 fis->lba_low_exp = (blkno >> 24) & 0xff;
3387 fis->lba_mid_exp = (blkno >> 32) & 0xff;
3388 fis->lba_high_exp = (blkno >> 40) & 0xff;
3389 fis->sector_count = sector_count & 0xff;
3390 fis->sector_count_exp = (sector_count >> 8) & 0xff;
3391 } else {
3392 /* Use LBA */
3393 fis->command = op == HIB_W1 ? ATA_C_WRITEDMA0xca : ATA_C_READDMA0xc8;
3394 fis->device = ATA_H2D_DEVICE_LBA0x40 | ((blkno >> 24) & 0x0f);
3395 fis->sector_count = sector_count & 0xff;
3396 }
3397
3398 my->ccb->ccb_xa.data = (void *)addr;
3399 my->ccb->ccb_xa.datalen = size;
3400 my->ccb->ccb_xa.pmp_port = my->pmp_port;
3401 my->ccb->ccb_xa.flags |= ATA_F_POLL(1<<3);
3402
3403 cmd_slot = my->ccb->ccb_cmd_hdr;
3404 flags = 5; /* FIS length (in DWORDs) */
3405 flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12;
3406
3407 if (op == HIB_W1)
3408 flags |= AHCI_CMD_LIST_FLAG_W(1<<6);
3409
3410 htolem16(&cmd_slot->flags, flags)(*(__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(flags
)))
;
3411
3412 ahci_hibernate_load_prdt(my->ccb);
3413
3414 ahci_hibernate_io_start(my->ap, my->ccb);
3415 timeout = 1000000;
3416 while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN35) {
3417 delay(1)(*delay_func)(1);
3418 timeout--;
3419 if (timeout == 0) {
3420 return (EIO5);
3421 }
3422 }
3423
3424 return (0);
3425}
3426
3427#endif