/usr/src/sys/dev/ic/ahci.c

Bug Summary

File:	dev/ic/ahci.c
Warning:	line 3418, column 10 Although the value stored to 'rc' is used in the enclosing expression, the value is never actually read from 'rc'

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name ahci.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/ic/ahci.c

1	/* $OpenBSD: ahci.c,v 1.37 2020/07/16 21:18:30 krw Exp $ */
2
3	/*
4	* Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
5	* Copyright (c) 2010 Conformal Systems LLC <info@conformal.com>
6	* Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org>
7	*
8	* Permission to use, copy, modify, and distribute this software for any
9	* purpose with or without fee is hereby granted, provided that the above
10	* copyright notice and this permission notice appear in all copies.
11	*
12	* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13	* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14	* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15	* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17	* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18	* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19	*/
20
21	#include <sys/param.h>
22	#include <sys/systm.h>
23	#include <sys/buf.h>
24	#include <sys/kernel.h>
25	#include <sys/malloc.h>
26	#include <sys/device.h>
27	#include <sys/queue.h>
28	#include <sys/mutex.h>
29	#include <sys/pool.h>
30
31	#include <machine/bus.h>
32
33	#include <dev/ic/ahcireg.h>
34	#include <dev/ic/ahcivar.h>
35
36	#ifdef AHCI_DEBUG
37	#define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
38	while (0)
39	#define AHCI_D_TIMEOUT 0x00
40	#define AHCI_D_VERBOSE 0x01
41	#define AHCI_D_INTR 0x02
42	#define AHCI_D_XFER 0x08
43	int ahcidebug = AHCI_D_VERBOSE;
44	#else
45	#define DPRINTF(m, f...)
46	#endif
47
48	#ifdef HIBERNATE1
49	#include <uvm/uvm_extern.h>
50	#include <sys/hibernate.h>
51	#include <sys/disk.h>
52	#include <sys/disklabel.h>
53
54	#include <scsi/scsi_all.h>
55	#include <scsi/scsiconf.h>
56
57	void ahci_hibernate_io_start(struct ahci_port *,
58	struct ahci_ccb *);
59	int ahci_hibernate_io_poll(struct ahci_port *,
60	struct ahci_ccb *);
61	void ahci_hibernate_load_prdt(struct ahci_ccb *);
62
63	int ahci_hibernate_io(dev_t dev, daddr_t blkno,
64	vaddr_t addr, size_t size, int wr, void *page);
65	#endif
66
67	struct cfdriver ahci_cd = {
68	NULL((void *)0), "ahci", DV_DULL
69	};
70
71	void ahci_enable_interrupts(struct ahci_port *);
72
73	int ahci_init(struct ahci_softc *);
74	int ahci_port_alloc(struct ahci_softc *, u_int);
75	void ahci_port_detect(struct ahci_softc *, u_int);
76	void ahci_port_free(struct ahci_softc *, u_int);
77	int ahci_port_init(struct ahci_softc *, u_int);
78
79	int ahci_default_port_start(struct ahci_port *, int);
80	int ahci_port_stop(struct ahci_port *, int);
81	int ahci_port_clo(struct ahci_port *);
82	int ahci_port_softreset(struct ahci_port *);
83	void ahci_port_comreset(struct ahci_port *);
84	int ahci_port_portreset(struct ahci_port *, int);
85	void ahci_port_portreset_start(struct ahci_port *);
86	int ahci_port_portreset_poll(struct ahci_port *);
87	void ahci_port_portreset_wait(struct ahci_port *);
88	int ahci_port_portreset_finish(struct ahci_port *, int);
89	int ahci_port_signature(struct ahci_port *);
90	int ahci_pmp_port_softreset(struct ahci_port *, int);
91	int ahci_pmp_port_portreset(struct ahci_port *, int);
92	int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
93
94	int ahci_load_prdt(struct ahci_ccb *);
95	void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
96	u_int32_t, u_int32_t);
97	void ahci_unload_prdt(struct ahci_ccb *);
98
99	int ahci_poll(struct ahci_ccb , int, void ()(void *));
100	void ahci_start(struct ahci_ccb *);
101
102	void ahci_issue_pending_ncq_commands(struct ahci_port *);
103	void ahci_issue_pending_commands(struct ahci_port *, int);
104
105	int ahci_intr(void *);
106	u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
107
108	struct ahci_ccb ahci_get_ccb(struct ahci_port );
109	void ahci_put_ccb(struct ahci_ccb *);
110
111	struct ahci_ccb ahci_get_err_ccb(struct ahci_port );
112	void ahci_put_err_ccb(struct ahci_ccb *);
113
114	struct ahci_ccb ahci_get_pmp_ccb(struct ahci_port );
115	void ahci_put_pmp_ccb(struct ahci_ccb *);
116
117	int ahci_port_read_ncq_error(struct ahci_port , int , int);
118
119	struct ahci_dmamem ahci_dmamem_alloc(struct ahci_softc , size_t);
120	void ahci_dmamem_free(struct ahci_softc *,
121	struct ahci_dmamem *);
122
123	u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
124	void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
125	int ahci_wait_ne(struct ahci_softc *, bus_size_t,
126	u_int32_t, u_int32_t);
127
128	u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
129	void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
130	int ahci_pwait_eq(struct ahci_port *, bus_size_t,
131	u_int32_t, u_int32_t, int);
132	void ahci_flush_tfd(struct ahci_port *ap);
133	u_int32_t ahci_active_mask(struct ahci_port *);
134	int ahci_port_detect_pmp(struct ahci_port *);
135	void ahci_pmp_probe_timeout(void *);
136
137	/* pmp operations */
138	int ahci_pmp_read(struct ahci_port *, int, int,
139	u_int32_t *);
140	int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
141	int ahci_pmp_phy_status(struct ahci_port *, int,
142	u_int32_t *);
143	int ahci_pmp_identify(struct ahci_port , int );
144
145
146	/* Wait for all bits in _b to be cleared */
147	#define ahci_pwait_clr(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) \
148	ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
149
150	/* Wait for all bits in _b to be set */
151	#define ahci_pwait_set(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) \
152	ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
153
154
155
156	/* provide methods for atascsi to call */
157	int ahci_ata_probe(void *, int, int);
158	void ahci_ata_free(void *, int, int);
159	struct ata_xfer * ahci_ata_get_xfer(void *, int);
160	void ahci_ata_put_xfer(struct ata_xfer *);
161	void ahci_ata_cmd(struct ata_xfer *);
162
163	struct atascsi_methods ahci_atascsi_methods = {
164	ahci_ata_probe,
165	ahci_ata_free,
166	ahci_ata_get_xfer,
167	ahci_ata_put_xfer,
168	ahci_ata_cmd
169	};
170
171	/* ccb completions */
172	void ahci_ata_cmd_done(struct ahci_ccb *);
173	void ahci_pmp_cmd_done(struct ahci_ccb *);
174	void ahci_ata_cmd_timeout(void *);
175	void ahci_empty_done(struct ahci_ccb *);
176
177	int
178	ahci_attach(struct ahci_softc *sc)
179	{
180	struct atascsi_attach_args aaa;
181	u_int32_t pi;
182	int i, j, done;
183
184	if (sc->sc_port_start == NULL((void *)0))
185	sc->sc_port_start = ahci_default_port_start;
186
187	if (ahci_init(sc) != 0) {
188	/* error already printed by ahci_init */
189	goto unmap;
190	}
191
192	printf("\n");
193
194	sc->sc_cap = ahci_read(sc, AHCI_REG_CAP0x000);
195	sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap)((((sc->sc_cap) & 0x1f00)>>8)+1);
196	#ifdef AHCI_DEBUG
197	if (ahcidebug & AHCI_D_VERBOSE) {
198	const char *gen;
199
200	switch (sc->sc_cap & AHCI_REG_CAP_ISS(0xf<<20)) {
201	case AHCI_REG_CAP_ISS_G1(0x1<<20):
202	gen = "1 (1.5Gbps)";
203	break;
204	case AHCI_REG_CAP_ISS_G2(0x2<<20):
205	gen = "2 (3.0Gb/s)";
206	break;
207	case AHCI_REG_CAP_ISS_G3(0x3<<20):
208	gen = "3 (6.0Gb/s)";
209	break;
210	default:
211	gen = "unknown";
212	break;
213	}
214
215	printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
216	DEVNAME(sc)((sc)->sc_dev.dv_xname), sc->sc_cap, AHCI_FMT_CAP"\020" "\040S64A" "\037NCQ" "\036SSNTF" "\035SMPS" "\034SSS" "\033SALP" "\032SAL" "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" "\021FBSS" "\020PMD" "\017SSC" "\016PSC" "\010CCCS" "\007EMS" "\006SXS",
217	AHCI_REG_CAP_NP(sc->sc_cap)(((sc->sc_cap) & 0x1f)+1), sc->sc_ncmds, gen);
218	printf("%s: extended capabilities 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
219	ahci_read(sc, AHCI_REG_CAP20x024), AHCI_FMT_CAP2"\020" "\006DESO" "\005SADM" "\004SDS" "\003APST" "\002NVMP" "\001BOH");
220	}
221	#endif
222
223	pi = ahci_read(sc, AHCI_REG_PI0x00c);
224	DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
225	DEVNAME(sc), pi);
226
227	#ifdef AHCI_COALESCE
228	/* Naive coalescing support - enable for all ports. */
229	if (sc->sc_cap & AHCI_REG_CAP_CCCS(1<<7)) {
230	u_int16_t ccc_timeout = 20;
231	u_int8_t ccc_numcomplete = 12;
232	u_int32_t ccc_ctl;
233
234	/* disable coalescing during reconfiguration. */
235	ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL0x014);
236	ccc_ctl &= ~0x00000001;
237	ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl);
238
239	sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl)(((ccc_ctl) & 0xf8) >> 3);
240	if (pi & sc->sc_ccc_mask) {
241	/* A conflict with the implemented port list? */
242	printf("%s: coalescing interrupt/implemented port list "
243	"conflict, PI: %08x, ccc_mask: %08x\n",
244	DEVNAME(sc)((sc)->sc_dev.dv_xname), pi, sc->sc_ccc_mask);
245	sc->sc_ccc_mask = 0;
246	goto noccc;
247	}
248
249	/* ahci_port_start will enable each port when it starts. */
250	sc->sc_ccc_ports = pi;
251	sc->sc_ccc_ports_cur = 0;
252
253	/* program thresholds and enable overall coalescing. */
254	ccc_ctl &= ~0xffffff00;
255	ccc_ctl \|= (ccc_timeout << 16) \| (ccc_numcomplete << 8);
256	ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl);
257	ahci_write(sc, AHCI_REG_CCC_PORTS0x018, 0);
258	ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl \| 1);
259	}
260	noccc:
261	#endif
262	/*
263	* Given that ahci_port_alloc() will grab one CCB for error recovery
264	* in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds
265	* pretend at least 2 command slots for devices without NCQ support.
266	* That way, also at least 1 slot is made available for atascsi(4).
267	*/
268	sc->sc_ncmds = max(2, sc->sc_ncmds);
269	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
270	if (!ISSET(pi, 1 << i)((pi) & (1 << i))) {
271	/* dont allocate stuff if the port isnt implemented */
272	continue;
273	}
274
275	if (ahci_port_alloc(sc, i) == ENOMEM12)
276	goto freeports;
277
278	if (sc->sc_ports[i] != NULL((void *)0))
279	ahci_port_portreset_start(sc->sc_ports[i]);
280	}
281
282	/*
283	* Poll for device detection until all ports report a device, or one
284	* second has elapsed.
285	*/
286	for (i = 0; i < 1000; i++) {
287	done = 1;
288	for (j = 0; j < AHCI_MAX_PORTS32; j++) {
289	if (sc->sc_ports[j] == NULL((void *)0))
290	continue;
291
292	if (ahci_port_portreset_poll(sc->sc_ports[j]))
293	done = 0;
294	}
295
296	if (done)
297	break;
298
299	delay(1000)(*delay_func)(1000);
300	}
301
302	/*
303	* Finish device detection on all ports that initialized.
304	*/
305	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
306	if (sc->sc_ports[i] != NULL((void *)0))
307	ahci_port_detect(sc, i);
308	}
309
310	memset(&aaa, 0, sizeof(aaa))__builtin_memset((&aaa), (0), (sizeof(aaa)));
311	aaa.aaa_cookie = sc;
312	aaa.aaa_methods = &ahci_atascsi_methods;
313	aaa.aaa_minphys = NULL((void *)0);
314	aaa.aaa_nports = AHCI_MAX_PORTS32;
315	aaa.aaa_ncmds = sc->sc_ncmds - 1;
316	if (!(sc->sc_flags & AHCI_F_NO_NCQ(1<<0)) &&
317	sc->sc_ncmds > 2 &&
318	(sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30))) {
319	aaa.aaa_capability \|= ASAA_CAP_NCQ(1 << 0) \| ASAA_CAP_PMP_NCQ(1 << 2);
320	}
321
322	sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
323
324	/* Enable interrupts */
325	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) \| AHCI_REG_GHC_IE(1<<1));
326
327	return 0;
328
329	freeports:
330	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
331	if (sc->sc_ports[i] != NULL((void *)0))
332	ahci_port_free(sc, i);
333	}
334	unmap:
335	/* Disable controller */
336	ahci_write(sc, AHCI_REG_GHC0x004, 0);
337	return 1;
338	}
339
340	int
341	ahci_detach(struct ahci_softc *sc, int flags)
342	{
343	int rv, i;
344
345	if (sc->sc_atascsi != NULL((void *)0)) {
346	rv = atascsi_detach(sc->sc_atascsi, flags);
347	if (rv != 0)
348	return (rv);
349	}
350
351	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
352	if (sc->sc_ports[i] != NULL((void *)0))
353	ahci_port_free(sc, i);
354	}
355
356	return (0);
357	}
358
359	int
360	ahci_activate(struct device *self, int act)
361	{
362	struct ahci_softc sc = (struct ahci_softc )self;
363	int i, rv = 0;
364
365	switch (act) {
366	case DVACT_RESUME4:
367	/* enable ahci (global interrupts disabled) */
368	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31));
369
370	/* restore BIOS initialised parameters */
371	ahci_write(sc, AHCI_REG_CAP0x000, sc->sc_cap);
372
373	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
374	if (sc->sc_ports[i] != NULL((void *)0))
375	ahci_port_init(sc, i);
376	}
377
378	/* Enable interrupts */
379	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) \| AHCI_REG_GHC_IE(1<<1));
380
381	rv = config_activate_children(self, act);
382	break;
383	case DVACT_POWERDOWN6:
384	rv = config_activate_children(self, act);
385	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
386	if (sc->sc_ports[i] != NULL((void *)0))
387	ahci_port_stop(sc->sc_ports[i], 1);
388	}
389	break;
390	default:
391	rv = config_activate_children(self, act);
392	break;
393	}
394	return (rv);
395	}
396
397	int
398	ahci_init(struct ahci_softc *sc)
399	{
400	u_int32_t reg, cap, pi;
401	const char *revision;
402
403	DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
404	AHCI_FMT_GHC);
405
406	/* save BIOS initialised parameters, enable staggered spin up */
407	cap = ahci_read(sc, AHCI_REG_CAP0x000);
408	cap &= AHCI_REG_CAP_SMPS(1<<28);
409	cap \|= AHCI_REG_CAP_SSS(1<<27);
410	pi = ahci_read(sc, AHCI_REG_PI0x00c);
411
412	if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))(((1<<31)) & (ahci_read(sc, 0x004)))) {
413	/* reset the controller */
414	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0));
415	if (ahci_wait_ne(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0),
416	AHCI_REG_GHC_HR(1<<0)) != 0) {
417	printf(" unable to reset controller\n");
418	return (1);
419	}
420	}
421
422	/* enable ahci (global interrupts disabled) */
423	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31));
424
425	/* restore parameters */
426	ahci_write(sc, AHCI_REG_CAP0x000, cap);
427	ahci_write(sc, AHCI_REG_PI0x00c, pi);
428
429	/* check the revision */
430	reg = ahci_read(sc, AHCI_REG_VS0x010);
431	switch (reg) {
432	case AHCI_REG_VS_0_950x00000905:
433	revision = "0.95";
434	break;
435	case AHCI_REG_VS_1_00x00010000:
436	revision = "1.0";
437	break;
438	case AHCI_REG_VS_1_10x00010100:
439	revision = "1.1";
440	break;
441	case AHCI_REG_VS_1_20x00010200:
442	revision = "1.2";
443	break;
444	case AHCI_REG_VS_1_30x00010300:
445	revision = "1.3";
446	break;
447	case AHCI_REG_VS_1_3_10x00010301:
448	revision = "1.3.1";
449	break;
450
451	default:
452	printf(" unsupported AHCI revision 0x%08x\n", reg);
453	return (1);
454	}
455
456	printf(" AHCI %s", revision);
457
458	return (0);
459	}
460
461	void
462	ahci_enable_interrupts(struct ahci_port *ap)
463	{
464	ahci_pwrite(ap, AHCI_PREG_IE0x14, AHCI_PREG_IE_TFEE(1<<30) \| AHCI_PREG_IE_HBFE(1<<29) \|
465	AHCI_PREG_IE_IFE(1<<27) \| AHCI_PREG_IE_OFE(1<<24) \| AHCI_PREG_IE_DPE(1<<5) \|
466	AHCI_PREG_IE_UFE(1<<4) \|
467	((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29)) ? AHCI_PREG_IE_IPME(1<<23) : 0) \|
468	#ifdef AHCI_COALESCE
469	((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
470	(AHCI_PREG_IE_SDBE(1<<3) \| AHCI_PREG_IE_DHRE(1<<0)))
471	#else
472	AHCI_PREG_IE_SDBE(1<<3) \| AHCI_PREG_IE_DHRE(1<<0)
473	#endif
474	);
475	}
476
477	int
478	ahci_port_alloc(struct ahci_softc *sc, u_int port)
479	{
480	struct ahci_port *ap;
481	struct ahci_ccb *ccb;
482	u_int64_t dva;
483	u_int32_t cmd;
484	struct ahci_cmd_hdr *hdr;
485	struct ahci_cmd_table *table;
486	int i, rc = ENOMEM12;
487
488	ap = malloc(sizeof(*ap), M_DEVBUF2, M_NOWAIT0x0002 \| M_ZERO0x0008);
489	if (ap == NULL((void *)0)) {
490	printf("%s: unable to allocate memory for port %d\n",
491	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
492	goto reterr;
493	}
494	ap->ap_err_scratch = dma_alloc(DEV_BSIZE(1 << 9), PR_NOWAIT0x0002 \| PR_ZERO0x0008);
495	if (ap->ap_err_scratch == NULL((void *)0)) {
496	printf("%s: unable to allocate DMA scratch buf for port %d\n",
497	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
498	free(ap, M_DEVBUF2, sizeof(*ap));
499	goto reterr;
500	}
501
502	#ifdef AHCI_DEBUG
503	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
504	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
505	#endif
506	ap->ap_port = port;
507	sc->sc_ports[port] = ap;
508
509	if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
510	AHCI_PORT_REGION(port)(0x100 + ((port) * 0x80)), AHCI_PORT_SIZE0x80, &ap->ap_ioh) != 0) {
511	printf("%s: unable to create register window for port %d\n",
512	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
513	goto freeport;
514	}
515
516	ap->ap_sc = sc;
517	#ifdef AHCI_COALESCE
518	ap->ap_num = port;
519	#endif
520	TAILQ_INIT(&ap->ap_ccb_free)do { (&ap->ap_ccb_free)->tqh_first = ((void *)0); ( &ap->ap_ccb_free)->tqh_last = &(&ap->ap_ccb_free )->tqh_first; } while (0);
521	TAILQ_INIT(&ap->ap_ccb_pending)do { (&ap->ap_ccb_pending)->tqh_first = ((void *)0) ; (&ap->ap_ccb_pending)->tqh_last = &(&ap-> ap_ccb_pending)->tqh_first; } while (0);
522	mtx_init(&ap->ap_ccb_mtx, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&ap-> ap_ccb_mtx), ((((0x6)) > 0x0 && ((0x6)) < 0x9) ? 0x9 : ((0x6)))); } while (0);
523
524	/* Disable port interrupts */
525	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
526
527	/* Sec 10.1.2 - deinitialise port if it is already running */
528	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
529	if (ISSET(cmd, (AHCI_PREG_CMD_ST \| AHCI_PREG_CMD_CR \|((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14))))
530	AHCI_PREG_CMD_FRE \| AHCI_PREG_CMD_FR))((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14)))) \|\|
531	ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) {
532	int r;
533
534	r = ahci_port_stop(ap, 1);
535	if (r) {
536	printf("%s: unable to disable %s, ignoring port %d\n",
537	DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port);
538	rc = ENXIO6;
539	goto freeport;
540	}
541
542	/* Write DET to zero */
543	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
544	}
545
546	/* Allocate RFIS */
547	ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
548	if (ap->ap_dmamem_rfis == NULL((void *)0))
549	goto nomem;
550
551	/* Setup RFIS base address */
552	ap->ap_rfis = (struct ahci_rfis ) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void )(ap->ap_dmamem_rfis)->adm_kva);
553	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0 ].ds_addr);
554	ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32));
555	ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva);
556
557	/* Enable FIS reception and activate port. */
558	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
559	cmd \|= AHCI_PREG_CMD_FRE(1<<4) \| AHCI_PREG_CMD_POD(1<<2) \| AHCI_PREG_CMD_SUD(1<<1);
560	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd \| AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
561
562	/* Check whether port activated. Skip it if not. */
563	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
564	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
565	rc = ENXIO6;
566	goto freeport;
567	}
568
569	/* Allocate a CCB for each command slot */
570	ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
571	M_DEVBUF2, M_NOWAIT0x0002 \| M_ZERO0x0008);
572	if (ap->ap_ccbs == NULL((void *)0)) {
573	printf("%s: unable to allocate command list for port %d\n",
574	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
575	goto freeport;
576	}
577
578	/* Command List Structures and Command Tables */
579	ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
580	sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
581	ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
582	sc->sc_ncmds * sizeof(struct ahci_cmd_table));
583	if (ap->ap_dmamem_cmd_table == NULL((void )0) \|\| ap->ap_dmamem_cmd_list == NULL((void )0)) {
584	nomem:
585	printf("%s: unable to allocate DMA memory for port %d\n",
586	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
587	goto freeport;
588	}
589
590	/* Setup command list base address */
591	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs [0].ds_addr);
592	ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32));
593	ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva);
594
595	/* Split CCB allocation into CCBs and assign to command header/table */
596	hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list)((void *)(ap->ap_dmamem_cmd_list)->adm_kva);
597	table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table)((void *)(ap->ap_dmamem_cmd_table)->adm_kva);
598	for (i = 0; i < sc->sc_ncmds; i++) {
599	ccb = &ap->ap_ccbs[i];
600
601	if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,((sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 \| 0x0002), (&ccb->ccb_dmamap))
602	(4 * 1024 * 1024), 0, BUS_DMA_NOWAIT \| BUS_DMA_ALLOCNOW,((sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 \| 0x0002), (&ccb->ccb_dmamap))
603	&ccb->ccb_dmamap)((sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 \| 0x0002), (&ccb->ccb_dmamap)) != 0) {
604	printf("%s: unable to create dmamap for port %d "
605	"ccb %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port, i);
606	goto freeport;
607	}
608
609	ccb->ccb_slot = i;
610	ccb->ccb_port = ap;
611	ccb->ccb_cmd_hdr = &hdr[i];
612	ccb->ccb_cmd_table = &table[i];
613	htolem64(&ccb->ccb_cmd_hdr->ctba,((__uint64_t )(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table ))))
614	AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +((__uint64_t )(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table ))))
615	ccb->ccb_slot * sizeof(struct ahci_cmd_table))((__uint64_t )(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table ))));
616
617	ccb->ccb_xa.fis =
618	(struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
619	ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
620	ccb->ccb_xa.tag = i;
621
622	ccb->ccb_xa.state = ATA_S_COMPLETE2;
623	ahci_put_ccb(ccb);
624	}
625
626	/* grab a ccb for use during error recovery */
627	ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
628	TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry)do { if (((ap->ap_ccb_err)->ccb_entry.tqe_next) != ((void )0)) (ap->ap_ccb_err)->ccb_entry.tqe_next->ccb_entry .tqe_prev = (ap->ap_ccb_err)->ccb_entry.tqe_prev; else ( &ap->ap_ccb_free)->tqh_last = (ap->ap_ccb_err)-> ccb_entry.tqe_prev; (ap->ap_ccb_err)->ccb_entry.tqe_prev = (ap->ap_ccb_err)->ccb_entry.tqe_next; ((ap->ap_ccb_err )->ccb_entry.tqe_prev) = ((void )-1); ((ap->ap_ccb_err )->ccb_entry.tqe_next) = ((void )-1); } while (0);
629	ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE2;
630
631	/* Wait for ICC change to complete */
632	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1));
633	rc = 0;
634
635	freeport:
636	if (rc != 0)
637	ahci_port_free(sc, port);
638	reterr:
639	return (rc);
640	}
641
642	void
643	ahci_port_detect(struct ahci_softc *sc, u_int port)
644	{
645	struct ahci_port *ap;
646	const char *speed;
647	int rc;
648
649	ap = sc->sc_ports[port];
650
651	rc = ahci_port_portreset_finish(ap, 1);
652	switch (rc) {
653	case ENODEV19:
654	switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
655	case AHCI_PREG_SSTS_DET_DEV_NE0x1:
656	printf("%s: device not communicating on port %d\n",
657	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
658	break;
659	case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4:
660	printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
661	port);
662	break;
663	default:
664	DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
665	"on port %d\n", DEVNAME(sc), port);
666	break;
667	}
668	goto freeport;
669
670	case EBUSY16:
671	printf("%s: device on port %d didn't come ready, "
672	"TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port,
673	ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
674
675	/* Try a soft reset to clear busy */
676	rc = ahci_port_softreset(ap);
677	if (rc) {
678	printf("%s: unable to communicate "
679	"with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
680	goto freeport;
681	}
682	break;
683
684	default:
685	break;
686	}
687
688	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
689	DEVNAME(sc), port, rc);
690
691	/* Read current link speed */
692	switch(ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_SPD0xf0) {
693	case AHCI_PREG_SSTS_SPD_GEN10x10:
694	speed = "1.5Gb/s";
695	break;
696	case AHCI_PREG_SSTS_SPD_GEN20x20:
697	speed = "3.0Gb/s";
698	break;
699	case AHCI_PREG_SSTS_SPD_GEN30x30:
700	speed = "6.0Gb/s";
701	break;
702	default:
703	speed = NULL((void *)0);
704	break;
705	}
706	if (speed != NULL((void *)0))
707	printf("%s: port %d: %s\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), port, speed);
708
709	/* Enable command transfers on port */
710	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
711	printf("%s: failed to start command DMA on port %d, "
712	"disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
713	rc = ENXIO6; /* couldn't start port */
714	}
715
716	/* Flush interrupts for port */
717	ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
718	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
719
720	ahci_enable_interrupts(ap);
721	freeport:
722	if (rc != 0)
723	ahci_port_free(sc, port);
724	}
725
726	void
727	ahci_port_free(struct ahci_softc *sc, u_int port)
728	{
729	struct ahci_port *ap = sc->sc_ports[port];
730	struct ahci_ccb *ccb;
731
732	/* Ensure port is disabled and its interrupts are flushed */
733	if (ap->ap_sc) {
734	ahci_pwrite(ap, AHCI_PREG_CMD0x18, 0);
735	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
736	ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
737	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
738	}
739
740	if (ap->ap_ccb_err)
741	ahci_put_ccb(ap->ap_ccb_err);
742
743	if (ap->ap_ccbs) {
744	while ((ccb = ahci_get_ccb(ap)) != NULL((void *)0))
745	bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (ccb ->ccb_dmamap));
746	free(ap->ap_ccbs, M_DEVBUF2, sc->sc_ncmds * sizeof(*ccb));
747	}
748
749	if (ap->ap_dmamem_cmd_list)
750	ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
751	if (ap->ap_dmamem_rfis)
752	ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
753	if (ap->ap_dmamem_cmd_table)
754	ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
755	if (ap->ap_err_scratch)
756	dma_free(ap->ap_err_scratch, DEV_BSIZE(1 << 9));
757
758	/* bus_space(9) says we dont free the subregions handle */
759
760	free(ap, M_DEVBUF2, sizeof(*ap));
761	sc->sc_ports[port] = NULL((void *)0);
762	}
763
764	int
765	ahci_port_init(struct ahci_softc *sc, u_int port)
766	{
767	struct ahci_port *ap;
768	u_int64_t dva;
769	u_int32_t cmd;
770	int rc = ENOMEM12;
771
772	ap = sc->sc_ports[port];
773	#ifdef AHCI_DEBUG
774	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
775	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
776	#endif
777
778	/* Disable port interrupts */
779	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
780
781	/* Sec 10.1.2 - deinitialise port if it is already running */
782	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
783	if (ISSET(cmd, (AHCI_PREG_CMD_ST \| AHCI_PREG_CMD_CR \|((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14))))
784	AHCI_PREG_CMD_FRE \| AHCI_PREG_CMD_FR))((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14)))) \|\|
785	ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) {
786	int r;
787
788	r = ahci_port_stop(ap, 1);
789	if (r) {
790	printf("%s: unable to disable %s, ignoring port %d\n",
791	DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port);
792	rc = ENXIO6;
793	goto reterr;
794	}
795
796	/* Write DET to zero */
797	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
798	}
799
800	/* Setup RFIS base address */
801	ap->ap_rfis = (struct ahci_rfis ) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void )(ap->ap_dmamem_rfis)->adm_kva);
802	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0 ].ds_addr);
803	ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32));
804	ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva);
805
806	/* Enable FIS reception and activate port. */
807	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
808	cmd \|= AHCI_PREG_CMD_FRE(1<<4) \| AHCI_PREG_CMD_POD(1<<2) \| AHCI_PREG_CMD_SUD(1<<1);
809	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd \| AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
810
811	/* Check whether port activated. Skip it if not. */
812	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
813	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
814	rc = ENXIO6;
815	goto reterr;
816	}
817
818	/* Setup command list base address */
819	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs [0].ds_addr);
820	ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32));
821	ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva);
822
823	/* Wait for ICC change to complete */
824	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1));
825
826	/* Reset port */
827	rc = ahci_port_portreset(ap, 1);
828	switch (rc) {
829	case ENODEV19:
830	switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
831	case AHCI_PREG_SSTS_DET_DEV_NE0x1:
832	printf("%s: device not communicating on port %d\n",
833	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
834	break;
835	case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4:
836	printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
837	port);
838	break;
839	default:
840	DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
841	"on port %d\n", DEVNAME(sc), port);
842	break;
843	}
844	goto reterr;
845
846	case EBUSY16:
847	printf("%s: device on port %d didn't come ready, "
848	"TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port,
849	ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
850
851	/* Try a soft reset to clear busy */
852	rc = ahci_port_softreset(ap);
853	if (rc) {
854	printf("%s: unable to communicate "
855	"with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
856	goto reterr;
857	}
858	break;
859
860	default:
861	break;
862	}
863	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
864	DEVNAME(sc), port);
865
866	if (ap->ap_pmp_ports > 0) {
867	int p;
868
869	for (p = 0; p < ap->ap_pmp_ports; p++) {
870	int sig;
871
872	/* might need to do a portreset first here? */
873
874	/* softreset the port */
875	if (ahci_pmp_port_softreset(ap, p)) {
876	printf("%s.%d: unable to probe PMP port due to"
877	" softreset failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p);
878	continue;
879	}
880
881	sig = ahci_port_signature(ap);
882	printf("%s.%d: port signature returned %d\n",
883	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p, sig);
884	}
885	}
886
887	/* Enable command transfers on port */
888	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
889	printf("%s: failed to start command DMA on port %d, "
890	"disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
891	rc = ENXIO6; /* couldn't start port */
892	}
893
894	/* Flush interrupts for port */
895	ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
896	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
897
898	ahci_enable_interrupts(ap);
899
900	reterr:
901	return (rc);
902	}
903
904	int
905	ahci_default_port_start(struct ahci_port *ap, int fre_only)
906	{
907	u_int32_t r;
908
909	/* Turn on FRE (and ST) */
910	r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
911	r \|= AHCI_PREG_CMD_FRE(1<<4);
912	if (!fre_only)
913	r \|= AHCI_PREG_CMD_ST(1<<0);
914	ahci_pwrite(ap, AHCI_PREG_CMD0x18, r);
915
916	#ifdef AHCI_COALESCE
917	/* (Re-)enable coalescing on the port. */
918	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
919	ap->ap_sc->sc_ccc_ports_cur \|= (1 << ap->ap_num);
920	ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018,
921	ap->ap_sc->sc_ccc_ports_cur);
922	}
923	#endif
924
925	return (0);
926	}
927
928	int
929	ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
930	{
931	u_int32_t r;
932
933	#ifdef AHCI_COALESCE
934	/* Disable coalescing on the port while it is stopped. */
935	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
936	ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
937	ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018,
938	ap->ap_sc->sc_ccc_ports_cur);
939	}
940	#endif
941
942	/* Turn off ST (and FRE) */
943	r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
944	r &= ~AHCI_PREG_CMD_ST(1<<0);
945	if (stop_fis_rx)
946	r &= ~AHCI_PREG_CMD_FRE(1<<4);
947	ahci_pwrite(ap, AHCI_PREG_CMD0x18, r);
948
949	/* Wait for CR to go off */
950	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)ahci_pwait_eq((ap), (0x18), ((1<<15)), 0, (1)))
951	return (1);
952
953	/* Wait for FR to go off */
954	if (stop_fis_rx &&
955	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)ahci_pwait_eq((ap), (0x18), ((1<<14)), 0, (1)))
956	return (2);
957
958	return (0);
959	}
960
961	/* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
962	int
963	ahci_port_clo(struct ahci_port *ap)
964	{
965	struct ahci_softc *sc = ap->ap_sc;
966	u_int32_t cmd;
967
968	/* Only attempt CLO if supported by controller */
969	if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(sc, 0x000)) & ((1<<24))))
970	return (1);
971
972	/* Issue CLO */
973	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
974	#ifdef DIAGNOSTIC1
975	if (ISSET(cmd, AHCI_PREG_CMD_ST)((cmd) & ((1<<0))))
976	printf("%s: CLO requested while port running\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
977	#endif
978	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd \| AHCI_PREG_CMD_CLO(1<<3));
979
980	/* Wait for completion */
981	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)ahci_pwait_eq((ap), (0x18), ((1<<3)), 0, (1))) {
982	printf("%s: CLO did not complete\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
983	return (1);
984	}
985
986	return (0);
987	}
988
989	/* AHCI soft reset, Section 10.4.1 */
990	int
991	ahci_port_softreset(struct ahci_port *ap)
992	{
993	struct ahci_ccb ccb = NULL((void )0);
994	struct ahci_cmd_hdr *cmd_slot;
995	u_int8_t *fis;
996	int s, rc = EIO5, oldstate;
997	u_int32_t cmd;
998
999	DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
1000
1001	s = splbio()splraise(0x6);
1002	oldstate = ap->ap_state;
1003	ap->ap_state = AP_S_ERROR_RECOVERY3;
1004
1005	/* Save previous command register state */
1006	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1007
1008	/* Idle port */
1009	if (ahci_port_stop(ap, 0)) {
1010	printf("%s: failed to stop port, cannot softreset\n",
1011	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1012	goto err;
1013	}
1014
1015	/* Request CLO if device appears hung */
1016	if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY \|((ahci_pread(ap, 0x20)) & ((1<<7) \| (1<<3)))
1017	AHCI_PREG_TFD_STS_DRQ)((ahci_pread(ap, 0x20)) & ((1<<7) \| (1<<3))))
1018	ahci_port_clo(ap);
1019
1020	/* Clear port errors to permit TFD transfer */
1021	ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30));
1022
1023	/* Restart port */
1024	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
1025	printf("%s: failed to start port, cannot softreset\n",
1026	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1027	goto err;
1028	}
1029
1030	/* Check whether CLO worked */
1031	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))
1032	AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))) {
1033	printf("%s: CLO %s, need port reset\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1034	ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(ap->ap_sc, 0x000)) & ((1<<24)))
1035	? "failed" : "unsupported");
1036	rc = EBUSY16;
1037	goto err;
1038	}
1039
1040	/* Prep first D2H command with SRST feature & clear busy/reset flags */
1041	ccb = ahci_get_err_ccb(ap);
1042	cmd_slot = ccb->ccb_cmd_hdr;
1043	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1044
1045	fis = ccb->ccb_cmd_table->cfis;
1046	fis[0] = ATA_FIS_TYPE_H2D0x27;
1047	fis[15] = ATA_FIS_CONTROL_SRST0x04;
1048
1049	cmd_slot->prdtl = 0;
1050	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (1<<6))))
1051	AHCI_CMD_LIST_FLAG_C \| AHCI_CMD_LIST_FLAG_R \|((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (1<<6))))
1052	AHCI_CMD_LIST_FLAG_W)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (1<<6))));
1053
1054	ccb->ccb_xa.state = ATA_S_PENDING1;
1055	if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0)
1056	goto err;
1057
1058	/* Prep second D2H command to read status and complete reset sequence */
1059	fis[0] = ATA_FIS_TYPE_H2D0x27;
1060	fis[15] = 0;
1061
1062	cmd_slot->prdtl = 0;
1063	htolem16(&cmd_slot->flags, 5 \| AHCI_CMD_LIST_FLAG_W)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<6))));
1064
1065	ccb->ccb_xa.state = ATA_S_PENDING1;
1066	if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0)
1067	goto err;
1068
1069	if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY \|ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (1))
1070	AHCI_PREG_TFD_STS_DRQ \| AHCI_PREG_TFD_STS_ERR, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (1))) {
1071	printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1072	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ahci_pread(ap, AHCI_PREG_TFD0x20),
1073	AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
1074	rc = EBUSY16;
1075	goto err;
1076	}
1077
1078	rc = 0;
1079	err:
1080	if (ccb != NULL((void *)0)) {
1081	/* Abort our command, if it failed, by stopping command DMA. */
1082	if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) {
1083	printf("%s: stopping the port, softreset slot %d was "
1084	"still active.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot);
1085	ahci_port_stop(ap, 0);
1086	}
1087	ccb->ccb_xa.state = ATA_S_ERROR3;
1088	ahci_put_err_ccb(ccb);
1089	}
1090
1091	/* Restore saved CMD register state */
1092	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1093	ap->ap_state = oldstate;
1094
1095	splx(s)spllower(s);
1096
1097	return (rc);
1098	}
1099
1100	int
1101	ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
1102	{
1103	struct ahci_ccb ccb = NULL((void )0);
1104	u_int32_t data;
1105	int count;
1106	int rc;
1107	int s;
1108	struct ahci_cmd_hdr *cmd_slot;
1109	u_int8_t *fis;
1110
1111	s = splbio()splraise(0x6);
1112	/* ignore spurious IFS errors while resetting */
1113	DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
1114	ap->ap_pmp_ignore_ifs = 1;
1115
1116	count = 2;
1117	rc = 0;
1118	do {
1119	if (ccb != NULL((void *)0)) {
1120	ahci_put_pmp_ccb(ccb);
1121	ccb = NULL((void *)0);
1122	}
1123
1124	if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1125	printf("%s.%d: unable to clear PHY status\n",
1126	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1127	}
1128	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1129	/* maybe don't do this on the first loop: */
1130	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1131	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1132
1133	/* send first softreset FIS */
1134	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1135	cmd_slot = ccb->ccb_cmd_hdr;
1136	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1137
1138	fis = ccb->ccb_cmd_table->cfis;
1139	fis[0] = ATA_FIS_TYPE_H2D0x27;
1140	fis[1] = pmp_port;
1141	fis[15] = ATA_FIS_CONTROL_SRST0x04 \| ATA_FIS_CONTROL_4BIT0x08;
1142
1143	cmd_slot->prdtl = 0;
1144	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (pmp_port << 12))))
1145	AHCI_CMD_LIST_FLAG_C \| AHCI_CMD_LIST_FLAG_R \|((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (pmp_port << 12))))
1146	(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (pmp_port << 12))));
1147
1148	ccb->ccb_xa.state = ATA_S_PENDING1;
1149
1150	DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
1151	PORTNAME(ap), pmp_port);
1152	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1153	printf("%s.%d: PMP port softreset cmd failed\n",
1154	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1155	rc = EBUSY16;
1156	if (count > 0) {
1157	/* probably delay a while to allow
1158	* it to settle down?
1159	*/
1160	}
1161	continue;
1162	}
1163
1164	/* send signature FIS */
1165	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1166	fis[0] = ATA_FIS_TYPE_H2D0x27;
1167	fis[1] = pmp_port;
1168	fis[15] = ATA_FIS_CONTROL_4BIT0x08;
1169
1170	cmd_slot->prdtl = 0;
1171	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))))
1172	(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))));
1173
1174	DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
1175	PORTNAME(ap), pmp_port);
1176	ccb->ccb_xa.state = ATA_S_PENDING1;
1177	if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1178	DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
1179	"failed\n", PORTNAME(ap), pmp_port);
1180	rc = EBUSY16;
1181	if (count > 0) {
1182	/* sleep a while? */
1183	}
1184	continue;
1185	}
1186
1187	fis[15] = 0;
1188	break;
1189	} while (count--);
1190
1191	if (ccb != NULL((void *)0)) {
1192	ahci_put_pmp_ccb(ccb);
1193	ccb = NULL((void *)0);
1194	}
1195
1196	/* clean up a bit */
1197	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1198	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1199	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1200	ap->ap_pmp_ignore_ifs = 0;
1201	DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
1202	splx(s)spllower(s);
1203
1204	return (rc);
1205	}
1206
1207	int
1208	ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
1209	{
1210	int sig;
1211
1212	ap->ap_state = AP_S_PMP_PORT_PROBE2;
1213
1214	DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
1215	pmp_port);
1216	if (ahci_pmp_port_portreset(ap, pmp_port)) {
1217	printf("%s.%d: unable to probe PMP port; portreset failed\n",
1218	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1219	ap->ap_state = AP_S_NORMAL0;
1220	return (ATA_PORT_T_NONE0);
1221	}
1222
1223	if (ahci_pmp_port_softreset(ap, pmp_port)) {
1224	printf("%s.%d: unable to probe PMP port due to softreset "
1225	"failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1226	ap->ap_state = AP_S_NORMAL0;
1227	return (ATA_PORT_T_NONE0);
1228	}
1229
1230	sig = ahci_port_signature(ap);
1231	DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
1232	PORTNAME(ap), pmp_port, sig);
1233	ap->ap_state = AP_S_NORMAL0;
1234	return (sig);
1235	}
1236
1237
1238	void
1239	ahci_flush_tfd(struct ahci_port *ap)
1240	{
1241	u_int32_t r;
1242
1243	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1244	if (r & AHCI_PREG_SERR_DIAG_X(1<<10))
1245	ahci_pwrite(ap, AHCI_PREG_SERR0x30, AHCI_PREG_SERR_DIAG_X(1<<10));
1246	}
1247
1248	u_int32_t
1249	ahci_active_mask(struct ahci_port *ap)
1250	{
1251	u_int32_t mask;
1252
1253	mask = ahci_pread(ap, AHCI_PREG_CI0x38);
1254	if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30))
1255	mask \|= ahci_pread(ap, AHCI_PREG_SACT0x34);
1256	return mask;
1257	}
1258
1259	void
1260	ahci_pmp_probe_timeout(void *cookie)
1261	{
1262	struct ahci_ccb *ccb = cookie;
1263	struct ahci_port *ap = ccb->ccb_port;
1264	u_int32_t mask;
1265
1266	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
1267	switch (ccb->ccb_xa.state) {
1268	case ATA_S_PENDING1:
1269	TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = ( ccb)->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = ( ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
1270	ccb->ccb_xa.state = ATA_S_TIMEOUT4;
1271	break;
1272
1273	case ATA_S_ONCHIP5:
1274	case ATA_S_ERROR3: /* currently mostly here for the ATI SBx00 quirk */
1275	/* clear the command on-chip */
1276	KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&((ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0" ))
1277	ap->ap_sactive == 0)((ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0" ));
1278	ahci_port_stop(ap, 0);
1279	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
1280
1281	if (ahci_active_mask(ap) != 0) {
1282	ahci_port_stop(ap, 0);
1283	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
1284	mask = ahci_active_mask(ap);
1285	if (mask != 0) {
1286	printf("%s: ahci_pmp_probe_timeout: failed to "
1287	"clear active cmds: %08x\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1288	mask);
1289	}
1290	}
1291
1292	ccb->ccb_xa.state = ATA_S_TIMEOUT4;
1293	ap->ap_active &= ~(1 << ccb->ccb_slot);
1294	KASSERT(ap->ap_active_cnt > 0)((ap->ap_active_cnt > 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1294, "ap->ap_active_cnt > 0" ));
1295	--ap->ap_active_cnt;
1296	DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, "
1297	"active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot,
1298	ap->ap_active, ap->ap_active_cnt);
1299	break;
1300
1301	default:
1302	panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
1303	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state);
1304	}
1305	}
1306
1307	int
1308	ahci_port_signature(struct ahci_port *ap)
1309	{
1310	u_int32_t sig;
1311
1312	sig = ahci_pread(ap, AHCI_PREG_SIG0x24);
1313	if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI0xeb140101 & 0xffff0000))
1314	return (ATA_PORT_T_ATAPI2);
1315	else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER0x96690101 &
1316	0xffff0000))
1317	return (ATA_PORT_T_PM3);
1318	else
1319	return (ATA_PORT_T_DISK1);
1320	}
1321
1322	int
1323	ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
1324	{
1325	u_int32_t cmd, data;
1326	int loop;
1327	int rc = 1;
1328	int s;
1329
1330	s = splbio()splraise(0x6);
1331	DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
1332	pmp_port);
1333
1334	/* Save previous command register state */
1335	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1336
1337	/* turn off power management and disable the PHY */
1338	data = AHCI_PREG_SCTL_IPM_DISABLED0x300;
1339	/* maybe add AHCI_PREG_SCTL_DET_DISABLE */
1340	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1))
1341	goto err;
1342	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1343	goto err;
1344	delay(10000)(*delay_func)(10000);
1345
1346	/* start COMRESET */
1347	data = AHCI_PREG_SCTL_IPM_DISABLED0x300 \| AHCI_PREG_SCTL_DET_INIT0x1;
1348	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1349	DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
1350	pmp_port);
1351	data \|= AHCI_PREG_SCTL_SPD_GEN10x10;
1352	} else
1353	data \|= AHCI_PREG_SCTL_SPD_ANY0x00;
1354
1355	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1356	goto err;
1357
1358	/* give it a while to settle down */
1359	delay(100000)(*delay_func)(100000);
1360
1361	if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1362	printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1363	pmp_port);
1364	}
1365
1366	/* start trying to negotiate */
1367	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1368	data = AHCI_PREG_SCTL_IPM_DISABLED0x300 \| AHCI_PREG_SCTL_DET_NONE0x0;
1369	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1370	goto err;
1371
1372	/* give it a while to detect */
1373	for (loop = 3; loop; --loop) {
1374	if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data))
1375	goto err;
1376	if (data & AHCI_PREG_SSTS_DET0xf)
1377	break;
1378	delay(100000)(*delay_func)(100000);
1379	}
1380	if (loop == 0) {
1381	printf("%s.%d: port is unplugged\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1382	goto err;
1383	}
1384
1385	/* give it even longer to fully negotiate */
1386	for (loop = 30; loop; --loop) {
1387	if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data))
1388	goto err;
1389	if ((data & AHCI_PREG_SSTS_DET0xf) == AHCI_PREG_SSTS_DET_DEV0x3)
1390	break;
1391	delay(100000)(*delay_func)(100000);
1392	}
1393
1394	if (loop == 0) {
1395	printf("%s.%d: device is not negotiating\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1396	pmp_port);
1397	goto err;
1398	}
1399
1400	/* device detected */
1401	DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
1402	pmp_port);
1403
1404	/* clean up a bit */
1405	delay(100000)(*delay_func)(100000);
1406	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1407	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1408	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1409
1410	rc = 0;
1411	err:
1412	/* Restore preserved port state */
1413	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1414	splx(s)spllower(s);
1415	return (rc);
1416	}
1417
1418	/* AHCI port reset, Section 10.4.2 */
1419
1420	void
1421	ahci_port_comreset(struct ahci_port *ap)
1422	{
1423	u_int32_t r;
1424
1425	r = AHCI_PREG_SCTL_IPM_DISABLED0x300 \| AHCI_PREG_SCTL_DET_INIT0x1;
1426	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1427	DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
1428	r \|= AHCI_PREG_SCTL_SPD_GEN10x10;
1429	} else
1430	r \|= AHCI_PREG_SCTL_SPD_ANY0x00;
1431	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r);
1432	delay(10000)(delay_func)(10000); / wait at least 1ms for COMRESET to be sent */
1433	r &= ~AHCI_PREG_SCTL_DET_INIT0x1;
1434	r \|= AHCI_PREG_SCTL_DET_NONE0x0;
1435	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r);
1436	delay(10000)(*delay_func)(10000);
1437	}
1438
1439	void
1440	ahci_port_portreset_start(struct ahci_port *ap)
1441	{
1442	int s;
1443
1444	s = splbio()splraise(0x6);
1445	DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1446
1447	/* Save previous command register state */
1448	ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1449
1450	/* Clear ST, ignoring failure */
1451	ahci_port_stop(ap, 0);
1452
1453	/* Perform device detection */
1454	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
1455	delay(10000)(*delay_func)(10000);
1456	ahci_port_comreset(ap);
1457	splx(s)spllower(s);
1458	}
1459
1460	int
1461	ahci_port_portreset_poll(struct ahci_port *ap)
1462	{
1463	if ((ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) !=
1464	AHCI_PREG_SSTS_DET_DEV0x3)
1465	return (EAGAIN35);
1466	return (0);
1467	}
1468
1469	void
1470	ahci_port_portreset_wait(struct ahci_port *ap)
1471	{
1472	int i;
1473
1474	for (i = 0; i < 1000; i++) {
1475	if (ahci_port_portreset_poll(ap) == 0)
1476	break;
1477	delay(1000)(*delay_func)(1000);
1478	}
1479	}
1480
1481	int
1482	ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
1483	{
1484	int rc, s, retries = 0;
1485
1486	s = splbio()splraise(0x6);
1487	retry:
1488	if (ahci_port_portreset_poll(ap)) {
1489	rc = ENODEV19;
1490	if (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
1491	/* this may be a port multiplier with no device
1492	* on port 0, so still do the pmp check if requested.
1493	*/
1494	} else {
1495	goto err;
1496	}
1497	} else {
1498	/* Clear SERR (incl X bit), so TFD can update */
1499	ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30));
1500
1501	/* Wait for device to become ready */
1502	if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY \|ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (3))
1503	AHCI_PREG_TFD_STS_DRQ \| AHCI_PREG_TFD_STS_ERR, 3)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (3))) {
1504	/* even if the device doesn't wake up, check if there's
1505	* a port multiplier there
1506	*/
1507	if (retries == 0) {
1508	retries = 1;
1509	ahci_port_comreset(ap);
1510	ahci_port_portreset_wait(ap);
1511	goto retry;
1512	}
1513	rc = EBUSY16;
1514	} else {
1515	rc = 0;
1516	}
1517	}
1518
1519	if (pmp != 0) {
1520	if (ahci_port_detect_pmp(ap)) {
1521	/* reset again without pmp support */
1522	pmp = 0;
1523	retries = 0;
1524	ahci_port_comreset(ap);
1525	ahci_port_portreset_wait(ap);
1526	goto retry;
1527	}
1528	}
1529
1530	err:
1531	/* Restore preserved port state */
1532	ahci_pwrite(ap, AHCI_PREG_CMD0x18, ap->ap_saved_cmd);
1533	ap->ap_saved_cmd = 0;
1534	splx(s)spllower(s);
1535
1536	return (rc);
1537	}
1538
1539	int
1540	ahci_port_portreset(struct ahci_port *ap, int pmp)
1541	{
1542	ahci_port_portreset_start(ap);
1543	ahci_port_portreset_wait(ap);
1544	return (ahci_port_portreset_finish(ap, pmp));
1545	}
1546
1547	int
1548	ahci_port_detect_pmp(struct ahci_port *ap)
1549	{
1550	int count, pmp_rc, rc;
1551	u_int32_t r, cmd;
1552	struct ahci_cmd_hdr *cmd_slot;
1553	struct ahci_ccb ccb = NULL((void )0);
1554	u_int8_t fis = NULL((void )0);
1555
1556	if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP(1<<2)) \|\|
1557	!ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)((ahci_read(ap->ap_sc, 0x000)) & ((1<<17)))) {
1558	return 0;
1559	}
1560
1561	rc = 0;
1562	pmp_rc = 0;
1563	count = 2;
1564	do {
1565	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
1566	count);
1567	if (ccb != NULL((void *)0)) {
1568	ahci_put_pmp_ccb(ccb);
1569	ccb = NULL((void *)0);
1570	}
1571	ahci_port_stop(ap, 0);
1572	ap->ap_state = AP_S_PMP_PROBE1;
1573
1574	/* set PMA in cmd reg */
1575	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1576	if ((cmd & AHCI_PREG_CMD_PMA(1<<17)) == 0) {
1577	cmd \|= AHCI_PREG_CMD_PMA(1<<17);
1578	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1579	}
1580
1581	/* Flush errors and request CLO unconditionally,
1582	* then start the port
1583	*/
1584	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1585	if (r & AHCI_PREG_SERR_DIAG_X(1<<10))
1586	ahci_pwrite(ap, AHCI_PREG_SERR0x30,
1587	AHCI_PREG_SERR_DIAG_X(1<<10));
1588
1589	/* Request CLO */
1590	ahci_port_clo(ap);
1591
1592	/* Clear port errors to permit TFD transfer */
1593	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1594	ahci_pwrite(ap, AHCI_PREG_SERR0x30, r);
1595
1596	/* Restart port */
1597	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
1598	rc = EBUSY16;
1599	printf("%s: failed to start port, cannot probe PMP\n",
1600	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1601	break;
1602	}
1603
1604	/* Check whether CLO worked */
1605	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))
1606	AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))) {
1607	u_int32_t cap;
1608
1609	cap = ahci_read(ap->ap_sc, AHCI_REG_CAP0x000);
1610	printf("%s: CLO %s, need port reset\n",
1611	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1612	ISSET(cap, AHCI_REG_CAP_SCLO)((cap) & ((1<<24)))
1613	? "failed" : "unsupported");
1614	pmp_rc = EBUSY16;
1615	break;
1616	}
1617
1618	/* Prep first command with SRST feature &
1619	* clear busy/reset flags
1620	*/
1621	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1622	cmd_slot = ccb->ccb_cmd_hdr;
1623	memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)))
1624	sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1625
1626	fis = ccb->ccb_cmd_table->cfis;
1627	fis[0] = ATA_FIS_TYPE_H2D0x27;
1628	fis[1] = SATA_PMP_CONTROL_PORT0x0f;
1629	fis[15] = ATA_FIS_CONTROL_SRST0x04 \| ATA_FIS_CONTROL_4BIT0x08;
1630
1631	cmd_slot->prdtl = 0;
1632	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| 0xf000)))
1633	AHCI_CMD_LIST_FLAG_C \| AHCI_CMD_LIST_FLAG_R \|((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| 0xf000)))
1634	AHCI_CMD_LIST_FLAG_PMP)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| 0xf000)));
1635
1636	DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
1637	PORTNAME(ap));
1638	ccb->ccb_xa.state = ATA_S_PENDING1;
1639	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1640	DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
1641	PORTNAME(ap));
1642	pmp_rc = EBUSY16;
1643	continue;
1644	}
1645
1646	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))
1647	AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))) {
1648	printf("%s: port busy after first PMP probe FIS\n",
1649	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1650	}
1651
1652	/* clear errors in case the device
1653	* didn't reset cleanly
1654	*/
1655	ahci_flush_tfd(ap);
1656	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1657	ahci_pwrite(ap, AHCI_PREG_SERR0x30, r);
1658
1659	/* Prep second command to read status and
1660	* complete reset sequence
1661	*/
1662	memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)))
1663	sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1664	fis[0] = ATA_FIS_TYPE_H2D0x27;
1665	fis[1] = SATA_PMP_CONTROL_PORT0x0f;
1666	fis[15] = ATA_FIS_CONTROL_4BIT0x08;
1667
1668	cmd_slot->prdtl = 0;
1669	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| 0xf000)))
1670	AHCI_CMD_LIST_FLAG_PMP)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| 0xf000)));
1671
1672	DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
1673	PORTNAME(ap));
1674	ccb->ccb_xa.state = ATA_S_PENDING1;
1675	if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1676	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
1677	"cmd failed\n", PORTNAME(ap));
1678	pmp_rc = EBUSY16;
1679	continue;
1680	}
1681
1682	/* apparently we need to retry at least once
1683	* to get the right signature
1684	*/
1685	fis[15] = 0;
1686	pmp_rc = 0;
1687	} while (--count);
1688
1689	if (ccb != NULL((void *)0)) {
1690	ahci_put_pmp_ccb(ccb);
1691	ccb = NULL((void *)0);
1692	}
1693
1694	if (ap->ap_state == AP_S_PMP_PROBE1) {
1695	ap->ap_state = AP_S_NORMAL0;
1696	}
1697
1698	if (pmp_rc == 0) {
1699	if (ahci_port_signature(ap) != ATA_PORT_T_PM3) {
1700	DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
1701	PORTNAME(ap));
1702	pmp_rc = EBUSY16;
1703	} else {
1704	DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
1705	PORTNAME(ap));
1706	}
1707	}
1708
1709	if (pmp_rc == 0) {
1710	if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
1711	pmp_rc = EBUSY16;
1712	} else {
1713	rc = 0;
1714	}
1715	}
1716
1717	/* if PMP detection failed, so turn off the PMA bit and
1718	* reset the port again
1719	*/
1720	if (pmp_rc != 0) {
1721	DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
1722	"the port\n", PORTNAME(ap));
1723	ahci_port_stop(ap, 0);
1724	ahci_port_clo(ap);
1725	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1726	cmd &= ~AHCI_PREG_CMD_PMA(1<<17);
1727	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1728
1729	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
1730	ahci_port_stop(ap, 0);
1731	if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29))
1732	ahci_pwrite(ap, AHCI_PREG_SNTF0x3c, -1);
1733	ahci_flush_tfd(ap);
1734	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1735
1736	ahci_pwrite(ap, AHCI_PREG_IS0x10, -1);
1737
1738	ahci_enable_interrupts(ap);
1739
1740	rc = pmp_rc;
1741	}
1742
1743	return (rc);
1744	}
1745
1746	void
1747	ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
1748	u_int32_t flags)
1749	{
1750	flags \|= len - 1;
1751
1752	htolem64(&prd->dba, addr)((__uint64_t )(&prd->dba) = ((__uint64_t)(addr)));
1753	htolem32(&prd->flags, flags)((__uint32_t )(&prd->flags) = ((__uint32_t)(flags)));
1754	}
1755
1756	int
1757	ahci_load_prdt(struct ahci_ccb *ccb)
1758	{
1759	struct ahci_port *ap = ccb->ccb_port;
1760	struct ahci_softc *sc = ap->ap_sc;
1761	struct ata_xfer *xa = &ccb->ccb_xa;
1762	struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
1763	bus_dmamap_t dmap = ccb->ccb_dmamap;
1764	struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1765	int i, error;
1766
1767	if (xa->datalen == 0) {
1768	ccb->ccb_cmd_hdr->prdtl = 0;
1769	return (0);
1770	}
1771
1772	error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap) , (xa->data), (xa->datalen), (((void )0)), ((xa->flags & (1<<2)) ? 0x0001 : 0x0000))
1773	(xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK)((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap) , (xa->data), (xa->datalen), (((void )0)), ((xa->flags & (1<<2)) ? 0x0001 : 0x0000));
1774	if (error != 0) {
1775	printf("%s: error %d loading dmamap\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), error);
1776	return (1);
1777	}
1778
1779	for (i = 0; i < dmap->dm_nsegs - 1; i++) {
1780	ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
1781	dmap->dm_segs[i].ds_len, 0);
1782	}
1783
1784	ahci_load_prdt_seg(&prdt[i],
1785	dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
1786	ISSET(xa->flags, ATA_F_PIO)((xa->flags) & ((1<<4))) ? AHCI_PRDT_FLAG_INTR(1<<31) : 0);
1787
1788	htolem16(&cmd_slot->prdtl, dmap->dm_nsegs)((__uint16_t )(&cmd_slot->prdtl) = ((__uint16_t)(dmap ->dm_nsegs)));
1789
1790	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04))
1791	(xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04))
1792	BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04));
1793
1794	return (0);
1795	}
1796
1797	void
1798	ahci_unload_prdt(struct ahci_ccb *ccb)
1799	{
1800	struct ahci_port *ap = ccb->ccb_port;
1801	struct ahci_softc *sc = ap->ap_sc;
1802	struct ata_xfer *xa = &ccb->ccb_xa;
1803	bus_dmamap_t dmap = ccb->ccb_dmamap;
1804
1805	if (xa->datalen != 0) {
1806	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08))
1807	(xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08))
1808	BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08));
1809
1810	bus_dmamap_unload(sc->sc_dmat, dmap)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (dmap ));
1811
1812	if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6))
1813	xa->resid = 0;
1814	else
1815	xa->resid = xa->datalen -
1816	lemtoh32(&ccb->ccb_cmd_hdr->prdbc)((__uint32_t)((__uint32_t )(&ccb->ccb_cmd_hdr->prdbc )));
1817	}
1818	}
1819
1820	int
1821	ahci_poll(struct ahci_ccb ccb, int timeout, void (timeout_fn)(void *))
1822	{
1823	struct ahci_port *ap = ccb->ccb_port;
1824	int s;
1825
1826	s = splbio()splraise(0x6);
1827	ahci_start(ccb);
1828	do {
1829	if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot ))
1830	1 << ccb->ccb_slot)((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot ))) {
1831	splx(s)spllower(s);
1832	return (0);
1833	}
1834	if (ccb->ccb_xa.state == ATA_S_ERROR3) {
1835	DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
1836	PORTNAME(ap), ccb->ccb_slot);
1837	/* pretend it timed out? */
1838	if (timeout_fn != NULL((void *)0)) {
1839	timeout_fn(ccb);
1840	}
1841	splx(s)spllower(s);
1842	return (1);
1843	}
1844
1845	delay(1000)(*delay_func)(1000);
1846	} while (--timeout > 0);
1847
1848	/* Run timeout while at splbio, otherwise ahci_intr could interfere. */
1849	if (timeout_fn != NULL((void *)0))
1850	timeout_fn(ccb);
1851
1852	splx(s)spllower(s);
1853
1854	return (1);
1855	}
1856
1857	void
1858	ahci_start(struct ahci_ccb *ccb)
1859	{
1860	struct ahci_port *ap = ccb->ccb_port;
1861	struct ahci_softc *sc = ap->ap_sc;
1862
1863	/* Zero transferred byte count before transfer */
1864	ccb->ccb_cmd_hdr->prdbc = 0;
1865
1866	/* Sync command list entry and corresponding command table entry */
1867	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) )
1868	ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) )
1869	sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) );
1870	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 ))
1871	ccb->ccb_slot * sizeof(struct ahci_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 ))
1872	sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 ));
1873
1874	/* Prepare RFIS area for write by controller */
1875	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x01))
1876	sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x01));
1877
1878	if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) {
1879	/* Issue NCQ commands only when there are no outstanding
1880	* standard commands. */
1881	if (ap->ap_active != 0 \|\| !TAILQ_EMPTY(&ap->ap_ccb_pending)(((&ap->ap_ccb_pending)->tqh_first) == ((void *)0)) \|\|
1882	(ap->ap_sactive != 0 &&
1883	ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
1884	TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_pending)->tqh_last; (& ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
1885	} else {
1886	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1886, "ap->ap_active_cnt == 0" ));
1887	ap->ap_sactive \|= (1 << ccb->ccb_slot);
1888	ccb->ccb_xa.state = ATA_S_ONCHIP5;
1889	ahci_pwrite(ap, AHCI_PREG_SACT0x34, 1 << ccb->ccb_slot);
1890	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
1891	ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
1892	}
1893	} else {
1894	/* Wait for all NCQ commands to finish before issuing standard
1895	* command. */
1896	if (ap->ap_sactive != 0 \|\| ap->ap_active_cnt == 2)
1897	TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_pending)->tqh_last; (& ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
1898	else if (ap->ap_active_cnt < 2) {
1899	ap->ap_active \|= 1 << ccb->ccb_slot;
1900	ccb->ccb_xa.state = ATA_S_ONCHIP5;
1901	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
1902	ap->ap_active_cnt++;
1903	}
1904	}
1905	}
1906
1907	void
1908	ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1909	{
1910	struct ahci_ccb *nextccb;
1911	u_int32_t sact_change = 0;
1912
1913	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1913, "ap->ap_active_cnt == 0" ));
1914
1915	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1916	if (nextccb == NULL((void *)0) \|\| !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)))
1917	return;
1918
1919	/* Start all the NCQ commands at the head of the pending list.
1920	* If a port multiplier is attached to the port, we can only
1921	* issue commands for one of its ports at a time.
1922	*/
1923	if (ap->ap_sactive != 0 &&
1924	ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
1925	return;
1926	}
1927
1928	ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
1929	do {
1930	TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void )0)) ( nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb )->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)-> tqh_last = (nextccb)->ccb_entry.tqe_prev; (nextccb)->ccb_entry .tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)-> ccb_entry.tqe_prev) = ((void )-1); ((nextccb)->ccb_entry. tqe_next) = ((void )-1); } while (0);
1931	sact_change \|= 1 << nextccb->ccb_slot;
1932	nextccb->ccb_xa.state = ATA_S_ONCHIP5;
1933	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1934	} while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) &&
1935	(nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
1936
1937	ap->ap_sactive \|= sact_change;
1938	ahci_pwrite(ap, AHCI_PREG_SACT0x34, sact_change);
1939	ahci_pwrite(ap, AHCI_PREG_CI0x38, sact_change);
1940	}
1941
1942	void
1943	ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
1944	{
1945	struct ahci_ccb *nextccb;
1946
1947	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1948	if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6))) {
1949	if (last_was_ncq) {
1950	KASSERT(nextccb->ccb_xa.pmp_port !=((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void )0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951 , "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port"))
1951	ap->ap_pmp_ncq_port)((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void )0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951 , "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port"));
1952	/* otherwise it should have been started already */
1953	} else {
1954	ap->ap_active_cnt--;
1955	}
1956
1957	/* Issue NCQ commands only when there are no outstanding
1958	* standard commands, and previous NCQ commands for other
1959	* PMP ports have finished.
1960	*/
1961	if (ap->ap_active == 0)
1962	ahci_issue_pending_ncq_commands(ap);
1963	else
1964	KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1964, "ap->ap_active_cnt == 1" ));
1965	} else if (nextccb) {
1966	if (ap->ap_sactive != 0 \|\| last_was_ncq)
1967	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1967, "ap->ap_active_cnt == 0" ));
1968
1969	/* Wait for all NCQ commands to finish before issuing standard
1970	* command. */
1971	if (ap->ap_sactive != 0)
1972	return;
1973
1974	/* Keep up to 2 standard commands on-chip at a time. */
1975	do {
1976	TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void )0)) ( nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb )->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)-> tqh_last = (nextccb)->ccb_entry.tqe_prev; (nextccb)->ccb_entry .tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)-> ccb_entry.tqe_prev) = ((void )-1); ((nextccb)->ccb_entry. tqe_next) = ((void )-1); } while (0);
1977	ap->ap_active \|= 1 << nextccb->ccb_slot;
1978	nextccb->ccb_xa.state = ATA_S_ONCHIP5;
1979	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << nextccb->ccb_slot);
1980	if (last_was_ncq)
1981	ap->ap_active_cnt++;
1982	if (ap->ap_active_cnt == 2)
1983	break;
1984	KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1984, "ap->ap_active_cnt == 1" ));
1985	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1986	} while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)));
1987	} else if (!last_was_ncq) {
1988	KASSERT(ap->ap_active_cnt == 1 \|\| ap->ap_active_cnt == 2)((ap->ap_active_cnt == 1 \|\| ap->ap_active_cnt == 2) ? ( void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1988, "ap->ap_active_cnt == 1 \|\| ap->ap_active_cnt == 2" ));
1989
1990	/* Standard command finished, none waiting to start. */
1991	ap->ap_active_cnt--;
1992	} else {
1993	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1993, "ap->ap_active_cnt == 0" ));
1994
1995	/* NCQ command finished. */
1996	}
1997	}
1998
1999	int
2000	ahci_intr(void *arg)
2001	{
2002	struct ahci_softc *sc = arg;
2003	u_int32_t is, ack = 0;
2004	int port;
2005
2006	/* Read global interrupt status */
2007	is = ahci_read(sc, AHCI_REG_IS0x008);
2008	if (is == 0 \|\| is == 0xffffffff)
2009	return (0);
2010	ack = is;
2011
2012	#ifdef AHCI_COALESCE
2013	/* Check coalescing interrupt first */
2014	if (is & sc->sc_ccc_mask) {
2015	DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2016	DEVNAME(sc));
2017	is &= ~sc->sc_ccc_mask;
2018	is \|= sc->sc_ccc_ports_cur;
2019	}
2020	#endif
2021
2022	/* Process interrupts for each port */
2023	while (is) {
2024	port = ffs(is) - 1;
2025	if (sc->sc_ports[port])
2026	ahci_port_intr(sc->sc_ports[port],
2027	AHCI_PREG_CI_ALL_SLOTS0xffffffff);
2028	is &= ~(1 << port);
2029	}
2030
2031	/* Finally, acknowledge global interrupt */
2032	ahci_write(sc, AHCI_REG_IS0x008, ack);
2033
2034	return (1);
2035	}
2036
2037	u_int32_t
2038	ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
2039	{
2040	struct ahci_softc *sc = ap->ap_sc;
2041	u_int32_t is, ci_saved, ci_masked, processed = 0;
2042	int slot, need_restart = 0;
2043	int process_error = 0;
2044	struct ahci_ccb *ccb;
2045	volatile u_int32_t *active;
2046	#ifdef DIAGNOSTIC1
2047	u_int32_t tmp;
2048	#endif
2049
2050	is = ahci_pread(ap, AHCI_PREG_IS0x10);
2051
2052	/* Ack port interrupt only if checking all command slots. */
2053	if (ci_mask == AHCI_PREG_CI_ALL_SLOTS0xffffffff)
2054	ahci_pwrite(ap, AHCI_PREG_IS0x10, is);
2055
2056	if (is)
2057	DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
2058	is, AHCI_PFMT_IS);
2059
2060	if (ap->ap_sactive) {
2061	/* Active NCQ commands - use SActive instead of CI */
2062	KASSERT(ap->ap_active == 0)((ap->ap_active == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2062, "ap->ap_active == 0"));
2063	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2063, "ap->ap_active_cnt == 0" ));
2064	ci_saved = ahci_pread(ap, AHCI_PREG_SACT0x34);
2065	active = &ap->ap_sactive;
2066	} else {
2067	/* Save CI */
2068	ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38);
2069	active = &ap->ap_active;
2070	}
2071
2072	if (is & AHCI_PREG_IS_TFES(1<<30)) {
2073	process_error = 1;
2074	} else if (is & AHCI_PREG_IS_DHRS(1<<0)) {
2075	u_int32_t tfd;
2076	u_int32_t cmd;
2077	u_int32_t serr;
2078
2079	tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
2080	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
2081	serr = ahci_pread(ap, AHCI_PREG_SERR0x30);
2082	if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) &&
2083	(cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) {
2084	DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
2085	" %b, DIAG: %b\n", PORTNAME(ap), tfd,
2086	AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2087	AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2088	AHCI_PFMT_SERR_DIAG);
2089	process_error = 1;
2090	} else {
2091	/* rfis copy back is in the normal execution path */
2092	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0));
2093	}
2094	}
2095
2096	/* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
2097	if (process_error) {
2098	u_int32_t tfd, serr;
2099	int err_slot;
2100
2101	tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
2102	serr = ahci_pread(ap, AHCI_PREG_SERR0x30);
2103
2104	if (ap->ap_sactive == 0) {
2105	/* Errored slot is easy to determine from CMD. */
2106	err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,(((ahci_pread(ap, 0x18)) >> 8) & 0x1f)
2107	AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f);
2108
2109	if ((ci_saved & (1 << err_slot)) == 0) {
2110	/*
2111	* Hardware doesn't seem to report correct
2112	* slot number. If there's only one
2113	* outstanding command we can cope,
2114	* otherwise fail all active commands.
2115	*/
2116	if (ap->ap_active_cnt == 1)
2117	err_slot = ffs(ap->ap_active) - 1;
2118	else
2119	goto failall;
2120	}
2121
2122	ccb = &ap->ap_ccbs[err_slot];
2123
2124	/* Preserve received taskfile data from the RFIS. */
2125	memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)))
2126	sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)));
2127	} else
2128	err_slot = -1; /* Must extract error from log page */
2129
2130	DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
2131	" %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
2132	AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2133	AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2134	AHCI_PFMT_SERR_DIAG);
2135
2136	/* Turn off ST to clear CI and SACT. */
2137	ahci_port_stop(ap, 0);
2138	need_restart = 1;
2139
2140	/* Clear SERR to enable capturing new errors. */
2141	ahci_pwrite(ap, AHCI_PREG_SERR0x30, serr);
2142
2143	/* Acknowledge the interrupts we can recover from. */
2144	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_TFES(1<<30) \|
2145	AHCI_PREG_IS_IFS(1<<27));
2146	is = ahci_pread(ap, AHCI_PREG_IS0x10);
2147
2148	/* If device hasn't cleared its busy status, try to idle it. */
2149	if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ)((tfd) & ((1<<7) \| (1<<3)))) {
2150
2151	if ((ap->ap_state == AP_S_PMP_PORT_PROBE2) \|\|
2152	(ap->ap_state == AP_S_ERROR_RECOVERY3)) {
2153	/* can't reset the port here, just make sure
2154	* the operation fails and the port still works.
2155	*/
2156	} else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
2157	printf("%s: error on PMP port %d, idling "
2158	"device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2159	ccb->ccb_xa.pmp_port);
2160	if (ahci_pmp_port_softreset(ap,
2161	ccb->ccb_xa.pmp_port) == 0) {
2162	printf("%s: unable to softreset port "
2163	"%d\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2164	ccb->ccb_xa.pmp_port);
2165	if (ahci_pmp_port_portreset(ap,
2166	ccb->ccb_xa.pmp_port)) {
2167	printf("%s: failed to port "
2168	" reset %d, giving up on "
2169	"it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2170	ccb->ccb_xa.pmp_port);
2171	goto fatal;
2172	}
2173	}
2174	} else {
2175	printf("%s: attempting to idle device\n",
2176	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2177	if (ahci_port_softreset(ap)) {
2178	printf("%s: failed to soft reset "
2179	"device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2180	if (ahci_port_portreset(ap, 0)) {
2181	printf("%s: failed to port "
2182	"reset device, give up on "
2183	"it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2184	goto fatal;
2185	}
2186	}
2187	}
2188
2189	/* Had to reset device, can't gather extended info. */
2190	} else if (ap->ap_sactive) {
2191	/* Recover the NCQ error from log page 10h.
2192	* We can only have queued commands active for one port
2193	* at a time, so we know which device errored.
2194	*/
2195	ahci_port_read_ncq_error(ap, &err_slot,
2196	ap->ap_pmp_ncq_port);
2197	if (err_slot < 0)
2198	goto failall;
2199
2200	DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2201	PORTNAME(ap), err_slot);
2202
2203	ccb = &ap->ap_ccbs[err_slot];
2204	if (ccb->ccb_xa.state != ATA_S_ONCHIP5) {
2205	printf("%s: NCQ errored slot %d is idle"
2206	" (%08x active)\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), err_slot,
2207	ci_saved);
2208	goto failall;
2209	}
2210	} else {
2211	/* Didn't reset, could gather extended info from log. */
2212	}
2213
2214	/*
2215	* If we couldn't determine the errored slot, reset the port
2216	* and fail all the active slots.
2217	*/
2218	if (err_slot == -1) {
2219	if (ahci_port_softreset(ap) != 0 &&
2220	ahci_port_portreset(ap, 0) != 0) {
2221	printf("%s: couldn't reset after NCQ error, "
2222	"disabling device.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2223	goto fatal;
2224	}
2225	printf("%s: couldn't recover NCQ error, failing "
2226	"all outstanding commands.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2227	goto failall;
2228	}
2229
2230	/* Clear the failed command in saved CI so completion runs. */
2231	ci_saved &= ~(1 << err_slot);
2232
2233	/* Note the error in the ata_xfer. */
2234	KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2234, "ccb->ccb_xa.state == ATA_S_ONCHIP" ));
2235	ccb->ccb_xa.state = ATA_S_ERROR3;
2236
2237	#ifdef DIAGNOSTIC1
2238	/* There may only be one outstanding standard command now. */
2239	if (ap->ap_sactive == 0) {
2240	tmp = ci_saved;
2241	if (tmp) {
2242	slot = ffs(tmp) - 1;
2243	tmp &= ~(1 << slot);
2244	KASSERT(tmp == 0)((tmp == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2244, "tmp == 0"));
2245	}
2246	}
2247	#endif
2248	}
2249
2250	/* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts
2251	* when there's a normal SATA device attached.
2252	*/
2253	if ((ap->ap_state == AP_S_PMP_PROBE1) &&
2254	(ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE(1<<1)) &&
2255	(is & AHCI_PREG_IS_IPMS(1<<23))) {
2256	slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f);
2257	DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
2258	PORTNAME(ap), slot);
2259
2260	ccb = &ap->ap_ccbs[slot];
2261	ccb->ccb_xa.state = ATA_S_ERROR3;
2262
2263	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IPMS(1<<23));
2264	is &= ~AHCI_PREG_IS_IPMS(1<<23);
2265	}
2266
2267	/* ignore IFS errors while resetting a PMP port */
2268	if ((is & AHCI_PREG_IS_IFS(1<<27)) /&& ap->ap_pmp_ignore_ifs/) {
2269	DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
2270	"port\n", PORTNAME(ap));
2271
2272	need_restart = 1;
2273	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
2274	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
2275	is &= ~AHCI_PREG_IS_IFS(1<<27);
2276	goto failall;
2277	}
2278
2279	/* Check for remaining errors - they are fatal. */
2280	if (is & (AHCI_PREG_IS_TFES(1<<30) \| AHCI_PREG_IS_HBFS(1<<29) \| AHCI_PREG_IS_IFS(1<<27) \|
2281	AHCI_PREG_IS_OFS(1<<24) \| AHCI_PREG_IS_UFS(1<<4))) {
2282	printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
2283	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), is, AHCI_PFMT_IS"\20" "\040CPDS" "\037TFES" "\036HBFS" "\035HBDS" "\034IFS" "\033INFS" "\031OFS" "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" "\007PCS" "\005UFS" "\004SDBS" "\003DSS" "\002PSS" "\001DHRS");
2284
2285	/* XXX try recovery first */
2286	goto fatal;
2287	}
2288
2289	/* Fail all outstanding commands if we know the port won't recover. */
2290	if (ap->ap_state == AP_S_FATAL_ERROR4) {
2291	fatal:
2292	ap->ap_state = AP_S_FATAL_ERROR4;
2293	failall:
2294
2295	/* Ensure port is shut down. */
2296	ahci_port_stop(ap, 1);
2297
2298	/* Error all the active slots. */
2299	ci_masked = ci_saved & *active;
2300	while (ci_masked) {
2301	slot = ffs(ci_masked) - 1;
2302	ccb = &ap->ap_ccbs[slot];
2303	ci_masked &= ~(1 << slot);
2304	ccb->ccb_xa.state = ATA_S_ERROR3;
2305	}
2306
2307	/* Run completion for all active slots. */
2308	ci_saved &= ~*active;
2309
2310	/* Don't restart the port if our problems were deemed fatal. */
2311	if (ap->ap_state == AP_S_FATAL_ERROR4)
2312	need_restart = 0;
2313	}
2314
2315	/*
2316	* CCB completion is detected by noticing its slot's bit in CI has
2317	* changed to zero some time after we activated it.
2318	* If we are polling, we may only be interested in particular slot(s).
2319	*/
2320	ci_masked = ~ci_saved & *active & ci_mask;
2321	while (ci_masked) {
2322	slot = ffs(ci_masked) - 1;
2323	ccb = &ap->ap_ccbs[slot];
2324	ci_masked &= ~(1 << slot);
2325
2326	DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2327	PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2328	" (error)" : "");
2329
2330	bus_dmamap_sync(sc->sc_dmat,((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) )
2331	AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) )
2332	ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) )
2333	sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) );
2334
2335	bus_dmamap_sync(sc->sc_dmat,((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ))
2336	AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ))
2337	ccb->ccb_slot * sizeof(struct ahci_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ))
2338	sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ));
2339
2340	bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02))
2341	AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02))
2342	sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02));
2343
2344	*active &= ~(1 << ccb->ccb_slot);
2345	/* Copy the rfis into the ccb if we were asked for it */
2346	if (ccb->ccb_xa.state == ATA_S_ONCHIP5 &&
2347	ccb->ccb_xa.flags & ATA_F_GET_RFIS(1<<8)) {
2348	memcpy(&ccb->ccb_xa.rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)))
2349	ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)))
2350	sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)));
2351	}
2352
2353	ccb->ccb_done(ccb);
2354
2355	processed \|= 1 << ccb->ccb_slot;
2356	}
2357
2358	if (need_restart) {
2359	/* Restart command DMA on the port */
2360	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
2361
2362	/* Re-enable outstanding commands on port. */
2363	if (ci_saved) {
2364	#ifdef DIAGNOSTIC1
2365	tmp = ci_saved;
2366	while (tmp) {
2367	slot = ffs(tmp) - 1;
2368	tmp &= ~(1 << slot);
2369	ccb = &ap->ap_ccbs[slot];
2370	KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2370, "ccb->ccb_xa.state == ATA_S_ONCHIP" ));
2371	KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap-> ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)" ))
2372	(!!ap->ap_sactive))(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap-> ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)" ));
2373	}
2374	#endif
2375	DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2376	"re-enabling%s slots %08x\n", PORTNAME(ap),
2377	ap->ap_sactive ? " NCQ" : "", ci_saved);
2378
2379	if (ap->ap_sactive)
2380	ahci_pwrite(ap, AHCI_PREG_SACT0x34, ci_saved);
2381	ahci_pwrite(ap, AHCI_PREG_CI0x38, ci_saved);
2382	}
2383	}
2384
2385	return (processed);
2386	}
2387
2388	struct ahci_ccb *
2389	ahci_get_ccb(struct ahci_port *ap)
2390	{
2391	struct ahci_ccb *ccb;
2392
2393	mtx_enter(&ap->ap_ccb_mtx);
2394	ccb = TAILQ_FIRST(&ap->ap_ccb_free)((&ap->ap_ccb_free)->tqh_first);
2395	if (ccb != NULL((void *)0)) {
2396	KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2396, "ccb->ccb_xa.state == ATA_S_PUT" ));
2397	TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb )->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = (ccb )->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = (( void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
2398	ccb->ccb_xa.state = ATA_S_SETUP0;
2399	}
2400	mtx_leave(&ap->ap_ccb_mtx);
2401
2402	return (ccb);
2403	}
2404
2405	void
2406	ahci_put_ccb(struct ahci_ccb *ccb)
2407	{
2408	struct ahci_port *ap = ccb->ccb_port;
2409
2410	#ifdef DIAGNOSTIC1
2411	if (ccb->ccb_xa.state != ATA_S_COMPLETE2 &&
2412	ccb->ccb_xa.state != ATA_S_TIMEOUT4 &&
2413	ccb->ccb_xa.state != ATA_S_ERROR3) {
2414	printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2415	"slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state,
2416	ccb->ccb_slot);
2417	}
2418	#endif
2419
2420	ccb->ccb_xa.state = ATA_S_PUT6;
2421	mtx_enter(&ap->ap_ccb_mtx);
2422	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_free)->tqh_last; (&ap ->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
2423	mtx_leave(&ap->ap_ccb_mtx);
2424	}
2425
2426	struct ahci_ccb *
2427	ahci_get_err_ccb(struct ahci_port *ap)
2428	{
2429	struct ahci_ccb *err_ccb;
2430	u_int32_t sact;
2431
2432	splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x6, __func__ ); } } while (0);
2433
2434	/* No commands may be active on the chip. */
2435	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2436	if (sact != 0)
2437	printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2438	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2438, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2439
2440	#ifdef DIAGNOSTIC1
2441	KASSERT(ap->ap_err_busy == 0)((ap->ap_err_busy == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2441, "ap->ap_err_busy == 0" ));
2442	ap->ap_err_busy = 1;
2443	#endif
2444	/* Save outstanding command state. */
2445	ap->ap_err_saved_active = ap->ap_active;
2446	ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2447	ap->ap_err_saved_sactive = ap->ap_sactive;
2448
2449	/*
2450	* Pretend we have no commands outstanding, so that completions won't
2451	* run prematurely.
2452	*/
2453	ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2454
2455	/*
2456	* Grab a CCB to use for error recovery. This should never fail, as
2457	* we ask atascsi to reserve one for us at init time.
2458	*/
2459	err_ccb = ap->ap_ccb_err;
2460	err_ccb->ccb_xa.flags = 0;
2461	err_ccb->ccb_xa.state = ATA_S_SETUP0;
2462	err_ccb->ccb_done = ahci_empty_done;
2463
2464	return (err_ccb);
2465	}
2466
2467	void
2468	ahci_put_err_ccb(struct ahci_ccb *ccb)
2469	{
2470	struct ahci_port *ap = ccb->ccb_port;
2471	u_int32_t sact;
2472
2473	splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x6, __func__ ); } } while (0);
2474
2475	#ifdef DIAGNOSTIC1
2476	KASSERT(ap->ap_err_busy)((ap->ap_err_busy) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2476, "ap->ap_err_busy"));
2477	#endif
2478	/* No commands may be active on the chip */
2479	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2480	if (sact != 0)
2481	printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact);
2482	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2482, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2483
2484	#ifdef DIAGNOSTIC1
2485	/* Done with the CCB */
2486	KASSERT(ccb == ap->ap_ccb_err)((ccb == ap->ap_ccb_err) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2486, "ccb == ap->ap_ccb_err" ));
2487	#endif
2488
2489	/* Restore outstanding command state */
2490	ap->ap_sactive = ap->ap_err_saved_sactive;
2491	ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2492	ap->ap_active = ap->ap_err_saved_active;
2493
2494	#ifdef DIAGNOSTIC1
2495	ap->ap_err_busy = 0;
2496	#endif
2497	}
2498
2499	struct ahci_ccb *
2500	ahci_get_pmp_ccb(struct ahci_port *ap)
2501	{
2502	struct ahci_ccb *ccb;
2503	u_int32_t sact;
2504
2505	/* some PMP commands need to be issued on slot 1,
2506	* particularly the command that clears SRST and
2507	* fetches the device signature.
2508	*
2509	* ensure the chip is idle and ccb 1 is available.
2510	*/
2511	splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x6, __func__ ); } } while (0);
2512
2513	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2514	if (sact != 0)
2515	printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
2516	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2516, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2517
2518	ccb = &ap->ap_ccbs[1];
2519	KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2519, "ccb->ccb_xa.state == ATA_S_PUT" ));
2520	ccb->ccb_xa.flags = 0;
2521	ccb->ccb_done = ahci_pmp_cmd_done;
2522
2523	mtx_enter(&ap->ap_ccb_mtx);
2524	TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb )->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = (ccb )->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = (( void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
2525	mtx_leave(&ap->ap_ccb_mtx);
2526
2527	return ccb;
2528	}
2529
2530	void
2531	ahci_put_pmp_ccb(struct ahci_ccb *ccb)
2532	{
2533	struct ahci_port *ap = ccb->ccb_port;
2534	u_int32_t sact;
2535
2536	/* make sure this is the right ccb */
2537	KASSERT(ccb == &ap->ap_ccbs[1])((ccb == &ap->ap_ccbs[1]) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2537, "ccb == &ap->ap_ccbs[1]" ));
2538
2539	/* No commands may be active on the chip */
2540	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2541	if (sact != 0)
2542	printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact);
2543	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2543, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2544
2545	ccb->ccb_xa.state = ATA_S_PUT6;
2546	mtx_enter(&ap->ap_ccb_mtx);
2547	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_free)->tqh_last; (&ap ->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
2548	mtx_leave(&ap->ap_ccb_mtx);
2549	}
2550
2551	int
2552	ahci_port_read_ncq_error(struct ahci_port ap, int err_slotp, int pmp_port)
2553	{
2554	struct ahci_ccb *ccb;
2555	struct ahci_cmd_hdr *cmd_slot;
2556	u_int32_t cmd;
2557	struct ata_fis_h2d *fis;
2558	int rc = EIO5, oldstate;
2559
2560	DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2561	oldstate = ap->ap_state;
2562	ap->ap_state = AP_S_ERROR_RECOVERY3;
2563
2564	/* Save command register state. */
2565	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
2566
2567	/* Port should have been idled already. Start it. */
2568	KASSERT((cmd & AHCI_PREG_CMD_CR) == 0)(((cmd & (1<<15)) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2568, "(cmd & AHCI_PREG_CMD_CR) == 0" ));
2569	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
2570
2571	/* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2572	ccb = ahci_get_err_ccb(ap);
2573	ccb->ccb_xa.flags = ATA_F_NOWAIT(1<<2) \| ATA_F_READ(1<<0) \| ATA_F_POLL(1<<3);
2574	ccb->ccb_xa.data = ap->ap_err_scratch;
2575	ccb->ccb_xa.datalen = 512;
2576	cmd_slot = ccb->ccb_cmd_hdr;
2577	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
2578
2579	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2580	fis->type = ATA_FIS_TYPE_H2D0x27;
2581	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| pmp_port;
2582	fis->command = ATA_C_READ_LOG_EXT0x2f;
2583	fis->lba_low = 0x10; /* queued error log page (10h) */
2584	fis->sector_count = 1; /* number of sectors (1) */
2585	fis->sector_count_exp = 0;
2586	fis->lba_mid = 0; /* starting offset */
2587	fis->lba_mid_exp = 0;
2588	fis->device = 0;
2589
2590	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))))
2591	(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))));
2592
2593	if (ahci_load_prdt(ccb) != 0) {
2594	rc = ENOMEM12; /* XXX caller must abort all commands */
2595	goto err;
2596	}
2597
2598	ccb->ccb_xa.state = ATA_S_PENDING1;
2599	if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0 \|\|
2600	ccb->ccb_xa.state == ATA_S_ERROR3)
2601	goto err;
2602
2603	rc = 0;
2604	err:
2605	/* Abort our command, if it failed, by stopping command DMA. */
2606	if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) {
2607	printf("%s: log page read failed, slot %d was still active.\n",
2608	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot);
2609	ahci_port_stop(ap, 0);
2610	}
2611
2612	/* Done with the error CCB now. */
2613	ahci_unload_prdt(ccb);
2614	ahci_put_err_ccb(ccb);
2615
2616	/* Extract failed register set and tags from the scratch space. */
2617	if (rc == 0) {
2618	struct ata_log_page_10h *log;
2619	int err_slot;
2620
2621	log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2622	if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)((log->err_regs.type) & (0x80))) {
2623	/* Not queued bit was set - wasn't an NCQ error? */
2624	printf("%s: read NCQ error page, but not an NCQ "
2625	"error?\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2626	rc = ESRCH3;
2627	} else {
2628	/* Copy back the log record as a D2H register FIS. */
2629	*err_slotp = err_slot = log->err_regs.type &
2630	ATA_LOG_10H_TYPE_TAG_MASK0x1f;
2631
2632	ccb = &ap->ap_ccbs[err_slot];
2633	memcpy(&ccb->ccb_xa.rfis, &log->err_regs,__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs ), (sizeof(struct ata_fis_d2h)))
2634	sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs ), (sizeof(struct ata_fis_d2h)));
2635	ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H0x34;
2636	ccb->ccb_xa.rfis.flags = 0;
2637	}
2638	}
2639
2640	/* Restore saved CMD register state */
2641	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
2642	ap->ap_state = oldstate;
2643
2644	return (rc);
2645	}
2646
2647	struct ahci_dmamem *
2648	ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
2649	{
2650	struct ahci_dmamem *adm;
2651	int nsegs;
2652
2653	adm = malloc(sizeof(*adm), M_DEVBUF2, M_NOWAIT0x0002 \| M_ZERO0x0008);
2654	if (adm == NULL((void *)0))
2655	return (NULL((void *)0));
2656
2657	adm->adm_size = size;
2658
2659	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size ), (1), (size), (0), (0x0001 \| 0x0002), (&adm->adm_map ))
2660	BUS_DMA_NOWAIT \| BUS_DMA_ALLOCNOW, &adm->adm_map)(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size ), (1), (size), (0), (0x0001 \| 0x0002), (&adm->adm_map )) != 0)
2661	goto admfree;
2662
2663	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size ), ((1 << 12)), (0), (&adm->adm_seg), (1), (& nsegs), (0x0001 \| 0x1000))
2664	1, &nsegs, BUS_DMA_NOWAIT \| BUS_DMA_ZERO)(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size ), ((1 << 12)), (0), (&adm->adm_seg), (1), (& nsegs), (0x0001 \| 0x1000)) != 0)
2665	goto destroy;
2666
2667	if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm ->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001 \| 0x0004))
2668	&adm->adm_kva, BUS_DMA_NOWAIT \| BUS_DMA_COHERENT)(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm ->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001 \| 0x0004)) != 0)
2669	goto free;
2670
2671	if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm-> adm_map), (adm->adm_kva), (size), (((void )0)), (0x0001))
2672	NULL, BUS_DMA_NOWAIT)((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm-> adm_map), (adm->adm_kva), (size), (((void )0)), (0x0001)) != 0)
2673	goto unmap;
2674
2675	return (adm);
2676
2677	unmap:
2678	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm-> adm_kva), (size));
2679	free:
2680	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (& adm->adm_seg), (1));
2681	destroy:
2682	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm ->adm_map));
2683	admfree:
2684	free(adm, M_DEVBUF2, sizeof(*adm));
2685
2686	return (NULL((void *)0));
2687	}
2688
2689	void
2690	ahci_dmamem_free(struct ahci_softc sc, struct ahci_dmamem adm)
2691	{
2692	bus_dmamap_unload(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (adm ->adm_map));
2693	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm-> adm_kva), (adm->adm_size));
2694	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (& adm->adm_seg), (1));
2695	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm ->adm_map));
2696	free(adm, M_DEVBUF2, sizeof(*adm));
2697	}
2698
2699	u_int32_t
2700	ahci_read(struct ahci_softc *sc, bus_size_t r)
2701	{
2702	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2703	BUS_SPACE_BARRIER_READ0x01);
2704	return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)((sc->sc_iot)->read_4((sc->sc_ioh), (r))));
2705	}
2706
2707	void
2708	ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2709	{
2710	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v)((sc->sc_iot)->write_4((sc->sc_ioh), (r), (v)));
2711	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2712	BUS_SPACE_BARRIER_WRITE0x02);
2713	}
2714
2715	int
2716	ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
2717	u_int32_t target)
2718	{
2719	int i;
2720
2721	for (i = 0; i < 1000; i++) {
2722	if ((ahci_read(sc, r) & mask) != target)
2723	return (0);
2724	delay(1000)(*delay_func)(1000);
2725	}
2726
2727	return (1);
2728	}
2729
2730	u_int32_t
2731	ahci_pread(struct ahci_port *ap, bus_size_t r)
2732	{
2733	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2734	BUS_SPACE_BARRIER_READ0x01);
2735	return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)((ap->ap_sc->sc_iot)->read_4((ap->ap_ioh), (r))));
2736	}
2737
2738	void
2739	ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2740	{
2741	bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v)((ap->ap_sc->sc_iot)->write_4((ap->ap_ioh), (r), ( v)));
2742	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2743	BUS_SPACE_BARRIER_WRITE0x02);
2744	}
2745
2746	int
2747	ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
2748	u_int32_t target, int n)
2749	{
2750	int i;
2751
2752	for (i = 0; i < n * 1000; i++) {
2753	if ((ahci_pread(ap, r) & mask) == target)
2754	return (0);
2755	delay(1000)(*delay_func)(1000);
2756	}
2757
2758	return (1);
2759	}
2760
2761	int
2762	ahci_ata_probe(void *xsc, int port, int lun)
2763	{
2764	struct ahci_softc *sc = xsc;
2765	struct ahci_port *ap = sc->sc_ports[port];
2766
2767	if (ap == NULL((void *)0))
2768	return (ATA_PORT_T_NONE0);
2769
2770	if (lun != 0) {
2771	int pmp_port = lun - 1;
2772	if (pmp_port >= ap->ap_pmp_ports) {
2773	return (ATA_PORT_T_NONE0);
2774	}
2775	return (ahci_pmp_port_probe(ap, pmp_port));
2776	} else {
2777	return (ahci_port_signature(ap));
2778	}
2779	}
2780
2781	void
2782	ahci_ata_free(void *xsc, int port, int lun)
2783	{
2784
2785	}
2786
2787	struct ata_xfer *
2788	ahci_ata_get_xfer(void *aaa_cookie, int port)
2789	{
2790	struct ahci_softc *sc = aaa_cookie;
2791	struct ahci_port *ap = sc->sc_ports[port];
2792	struct ahci_ccb *ccb;
2793
2794	ccb = ahci_get_ccb(ap);
2795	if (ccb == NULL((void *)0)) {
2796	DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
2797	PORTNAME(ap));
2798	return (NULL((void *)0));
2799	}
2800
2801	DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
2802	PORTNAME(ap), ccb->ccb_slot);
2803
2804	return ((struct ata_xfer *)ccb);
2805	}
2806
2807	void
2808	ahci_ata_put_xfer(struct ata_xfer *xa)
2809	{
2810	struct ahci_ccb ccb = (struct ahci_ccb )xa;
2811
2812	DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
2813
2814	ahci_put_ccb(ccb);
2815	}
2816
2817	void
2818	ahci_ata_cmd(struct ata_xfer *xa)
2819	{
2820	struct ahci_ccb ccb = (struct ahci_ccb )xa;
2821	struct ahci_cmd_hdr *cmd_slot;
2822	int s;
2823	u_int16_t flags;
2824
2825	if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR4)
2826	goto failcmd;
2827
2828	ccb->ccb_done = ahci_ata_cmd_done;
2829
2830	cmd_slot = ccb->ccb_cmd_hdr;
2831	flags = 5 /* FIS length (in DWORDs) */;
2832	flags \|= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12;
2833
2834	if (xa->flags & ATA_F_WRITE(1<<1))
2835	flags \|= AHCI_CMD_LIST_FLAG_W(1<<6);
2836
2837	if (xa->flags & ATA_F_PACKET(1<<5))
2838	flags \|= AHCI_CMD_LIST_FLAG_A(1<<5);
2839
2840	htolem16(&cmd_slot->flags, flags)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(flags )));
2841
2842	if (ahci_load_prdt(ccb) != 0)
2843	goto failcmd;
2844
2845	timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
2846
2847	xa->state = ATA_S_PENDING1;
2848
2849	if (xa->flags & ATA_F_POLL(1<<3))
2850	ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
2851	else {
2852	s = splbio()splraise(0x6);
2853	timeout_add_msec(&xa->stimeout, xa->timeout);
2854	ahci_start(ccb);
2855	splx(s)spllower(s);
2856	}
2857
2858	return;
2859
2860	failcmd:
2861	s = splbio()splraise(0x6);
2862	xa->state = ATA_S_ERROR3;
2863	ata_complete(xa);
2864	splx(s)spllower(s);
2865	}
2866
2867	void
2868	ahci_pmp_cmd_done(struct ahci_ccb *ccb)
2869	{
2870	struct ata_xfer *xa = &ccb->ccb_xa;
2871
2872	if (xa->state == ATA_S_ONCHIP5 \|\| xa->state == ATA_S_ERROR3)
2873	ahci_issue_pending_commands(ccb->ccb_port,
2874	xa->flags & ATA_F_NCQ(1<<6));
2875
2876	xa->state = ATA_S_COMPLETE2;
2877	}
2878
2879
2880	void
2881	ahci_ata_cmd_done(struct ahci_ccb *ccb)
2882	{
2883	struct ata_xfer *xa = &ccb->ccb_xa;
2884
2885	timeout_del(&xa->stimeout);
2886
2887	if (xa->state == ATA_S_ONCHIP5 \|\| xa->state == ATA_S_ERROR3)
2888	ahci_issue_pending_commands(ccb->ccb_port,
2889	xa->flags & ATA_F_NCQ(1<<6));
2890
2891	ahci_unload_prdt(ccb);
2892
2893	if (xa->state == ATA_S_ONCHIP5)
2894	xa->state = ATA_S_COMPLETE2;
2895	#ifdef DIAGNOSTIC1
2896	else if (xa->state != ATA_S_ERROR3 && xa->state != ATA_S_TIMEOUT4)
2897	printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
2898	"slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), xa->state,
2899	ccb->ccb_slot);
2900	#endif
2901	if (xa->state != ATA_S_TIMEOUT4)
2902	ata_complete(xa);
2903	}
2904
2905	void
2906	ahci_ata_cmd_timeout(void *arg)
2907	{
2908	struct ahci_ccb *ccb = arg;
2909	struct ata_xfer *xa = &ccb->ccb_xa;
2910	struct ahci_port *ap = ccb->ccb_port;
2911	int s, ccb_was_started, ncq_cmd;
2912	volatile u_int32_t *active;
2913
2914	s = splbio()splraise(0x6);
2915
2916	ncq_cmd = (xa->flags & ATA_F_NCQ(1<<6));
2917	active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
2918
2919	if (ccb->ccb_xa.state == ATA_S_PENDING1) {
2920	DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
2921	"before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
2922	TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = ( ccb)->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = ( ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
2923	ccb_was_started = 0;
2924	} else if (ccb->ccb_xa.state == ATA_S_ONCHIP5 && ahci_port_intr(ap,
2925	1 << ccb->ccb_slot)) {
2926	DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
2927	"command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
2928	goto ret;
2929	} else if (ccb->ccb_xa.state != ATA_S_ONCHIP5) {
2930	DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
2931	"handled%s\n", PORTNAME(ap), ccb->ccb_slot,
2932	ISSET(*active, 1 << ccb->ccb_slot) ?
2933	" but slot is still active?" : ".");
2934	goto ret;
2935	} else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb ->ccb_slot))
2936	AHCI_PREG_CI), 1 << ccb->ccb_slot)((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb ->ccb_slot)) && ISSET(active,((active) & (1 << ccb->ccb_slot))
2937	1 << ccb->ccb_slot)((*active) & (1 << ccb->ccb_slot))) {
2938	DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
2939	"IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
2940	ccb->ccb_slot);
2941	*active &= ~(1 << ccb->ccb_slot);
2942	ccb->ccb_done(ccb);
2943	goto ret;
2944	} else {
2945	ccb_was_started = 1;
2946	}
2947
2948	/* Complete the slot with a timeout error. */
2949	ccb->ccb_xa.state = ATA_S_TIMEOUT4;
2950	*active &= ~(1 << ccb->ccb_slot);
2951	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
2952	ccb->ccb_done(ccb); /* This won't issue pending commands or run the
2953	atascsi completion. */
2954
2955	/* Reset port to abort running command. */
2956	if (ccb_was_started) {
2957	DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
2958	"in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
2959	ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
2960	if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
2961	!= 0) {
2962	printf("%s: failed to reset port during timeout "
2963	"handling, disabling it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2964	ap->ap_state = AP_S_FATAL_ERROR4;
2965	}
2966
2967	/* Restart any other commands that were aborted by the reset. */
2968	if (*active) {
2969	DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
2970	"%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
2971	*active);
2972	if (ncq_cmd)
2973	ahci_pwrite(ap, AHCI_PREG_SACT0x34, *active);
2974	ahci_pwrite(ap, AHCI_PREG_CI0x38, *active);
2975	}
2976	}
2977
2978	/* Issue any pending commands now. */
2979	DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
2980	if (ccb_was_started)
2981	ahci_issue_pending_commands(ap, ncq_cmd);
2982	else if (ap->ap_active == 0)
2983	ahci_issue_pending_ncq_commands(ap);
2984
2985	/* Complete the timed out ata_xfer I/O (may generate new I/O). */
2986	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
2987	ata_complete(xa);
2988
2989	DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
2990	ret:
2991	splx(s)spllower(s);
2992	}
2993
2994	void
2995	ahci_empty_done(struct ahci_ccb *ccb)
2996	{
2997	if (ccb->ccb_xa.state != ATA_S_ERROR3)
2998	ccb->ccb_xa.state = ATA_S_COMPLETE2;
2999	}
3000
3001	int
3002	ahci_pmp_read(struct ahci_port ap, int target, int which, u_int32_t datap)
3003	{
3004	struct ahci_ccb *ccb;
3005	struct ata_fis_h2d *fis;
3006	int error;
3007
3008	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3009	ccb->ccb_xa.flags = ATA_F_POLL(1<<3) \| ATA_F_GET_RFIS(1<<8);
3010	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f;
3011	ccb->ccb_xa.state = ATA_S_PENDING1;
3012
3013	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
3014	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3015	fis->type = ATA_FIS_TYPE_H2D0x27;
3016	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| SATA_PMP_CONTROL_PORT0x0f;
3017	fis->command = ATA_C_READ_PM0xe4;
3018	fis->features = which;
3019	fis->device = target \| ATA_H2D_DEVICE_LBA0x40;
3020	fis->control = ATA_FIS_CONTROL_4BIT0x08;
3021
3022	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
3023	error = 1;
3024	} else {
3025	*datap = ccb->ccb_xa.rfis.sector_count \|
3026	(ccb->ccb_xa.rfis.lba_low << 8) \|
3027	(ccb->ccb_xa.rfis.lba_mid << 16) \|
3028	(ccb->ccb_xa.rfis.lba_high << 24);
3029	error = 0;
3030	}
3031	ahci_put_pmp_ccb(ccb);
3032	return (error);
3033	}
3034
3035	int
3036	ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
3037	{
3038	struct ahci_ccb *ccb;
3039	struct ata_fis_h2d *fis;
3040	int error;
3041
3042	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3043	ccb->ccb_xa.flags = ATA_F_POLL(1<<3);
3044	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f;
3045	ccb->ccb_xa.state = ATA_S_PENDING1;
3046
3047	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
3048	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3049	fis->type = ATA_FIS_TYPE_H2D0x27;
3050	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| SATA_PMP_CONTROL_PORT0x0f;
3051	fis->command = ATA_C_WRITE_PM0xe8;
3052	fis->features = which;
3053	fis->device = target \| ATA_H2D_DEVICE_LBA0x40;
3054	fis->sector_count = (u_int8_t)data;
3055	fis->lba_low = (u_int8_t)(data >> 8);
3056	fis->lba_mid = (u_int8_t)(data >> 16);
3057	fis->lba_high = (u_int8_t)(data >> 24);
3058	fis->control = ATA_FIS_CONTROL_4BIT0x08;
3059
3060	error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
3061	ahci_put_pmp_ccb(ccb);
3062	return (error);
3063	}
3064
3065	int
3066	ahci_pmp_phy_status(struct ahci_port ap, int target, u_int32_t datap)
3067	{
3068	int error;
3069
3070	error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS0, datap);
3071	if (error == 0)
3072	error = ahci_pmp_write(ap, target, SATA_PMREG_SERR1, -1);
3073	if (error)
3074	*datap = 0;
3075
3076	return (error);
3077	}
3078
3079	int
3080	ahci_pmp_identify(struct ahci_port ap, int ret_nports)
3081	{
3082	u_int32_t chipid;
3083	u_int32_t rev;
3084	u_int32_t nports;
3085	u_int32_t features;
3086	u_int32_t enabled;
3087	int s;
3088
3089	s = splbio()splraise(0x6);
3090
3091	if (ahci_pmp_read(ap, 15, 0, &chipid) \|\|
3092	ahci_pmp_read(ap, 15, 1, &rev) \|\|
3093	ahci_pmp_read(ap, 15, 2, &nports) \|\|
3094	ahci_pmp_read(ap, 15, SATA_PMREG_FEA64, &features) \|\|
3095	ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN96, &enabled)) {
3096	printf("%s: port multiplier identification failed\n",
3097	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
3098	splx(s)spllower(s);
3099	return (1);
3100	}
3101	splx(s)spllower(s);
3102
3103	nports &= 0x0F;
3104
3105	/* ignore SEMB port on SiI3726 port multiplier chips */
3106	if (chipid == 0x37261095) {
3107	nports--;
3108	}
3109
3110	printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
3111	"features: 0x%b, enabled: 0x%b\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), chipid, rev,
3112	SATA_PFMT_PM_REV"\20" "\003PM1.1" "\002PM1.0", nports, features, SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST", enabled,
3113	SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST");
3114
3115	*ret_nports = nports;
3116	return (0);
3117	}
3118
3119
3120	#ifdef HIBERNATE1
3121	void
3122	ahci_hibernate_io_start(struct ahci_port ap, struct ahci_ccb ccb)
3123	{
3124	ccb->ccb_cmd_hdr->prdbc = 0;
3125	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
3126	}
3127
3128	int
3129	ahci_hibernate_io_poll(struct ahci_port ap, struct ahci_ccb ccb)
3130	{
3131	u_int32_t is, ci_saved;
3132	int process_error = 0;
3133
3134	is = ahci_pread(ap, AHCI_PREG_IS0x10);
3135
3136	ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38);
3137
3138	if (is & AHCI_PREG_IS_DHRS(1<<0)) {
3139	u_int32_t tfd;
3140	u_int32_t cmd;
3141
3142	tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
3143	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
3144	if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) &&
3145	(cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) {
3146	process_error = 1;
3147	} else {
3148	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0));
3149	}
3150	} else if (is & (AHCI_PREG_IS_TFES(1<<30) \| AHCI_PREG_IS_HBFS(1<<29) \|
3151	AHCI_PREG_IS_IFS(1<<27) \| AHCI_PREG_IS_OFS(1<<24) \| AHCI_PREG_IS_UFS(1<<4))) {
3152	process_error = 1;
3153	}
3154
3155	/* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
3156	if (process_error) {
3157
3158	/* Turn off ST to clear CI and SACT. */
3159	ahci_port_stop(ap, 0);
3160
3161	/* just return an error indicator? we can't meaningfully
3162	* recover, and on the way back out we'll DVACT_RESUME which
3163	* resets and reinits the port.
3164	*/
3165	return (EIO5);
3166	}
3167
3168	/* command is finished when the bit in CI for the slot goes to 0 */
3169	if (ci_saved & (1 << ccb->ccb_slot)) {
3170	return (EAGAIN35);
3171	}
3172
3173	return (0);
3174	}
3175
3176	void
3177	ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
3178	{
3179	struct ata_xfer *xa = &ccb->ccb_xa;
3180	struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
3181	struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
3182	int i;
3183	paddr_t data_phys;
3184	u_int64_t data_bus_phys;
3185	vaddr_t data_addr;
3186	size_t seglen;
3187	size_t buflen;
3188
3189	if (xa->datalen == 0) {
3190	ccb->ccb_cmd_hdr->prdtl = 0;
3191	return;
3192	}
3193
3194	/* derived from i386/amd64 _bus_dma_load_buffer;
3195	* for amd64 the buffer will always be dma safe.
3196	*/
3197
3198	buflen = xa->datalen;
3199	data_addr = (vaddr_t)xa->data;
3200	for (i = 0; buflen > 0; i++) {
3201	pmap_extract(pmap_kernel()(&kernel_pmap_store), data_addr, &data_phys);
3202	data_bus_phys = data_phys;
3203
3204	seglen = PAGE_SIZE(1 << 12) - ((u_long)data_addr & PGOFSET((1 << 12) - 1));
3205	if (buflen < seglen)
3206	seglen = buflen;
3207
3208	ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
3209
3210	data_addr += seglen;
3211	buflen -= seglen;
3212	}
3213
3214	htolem16(&cmd_slot->prdtl, i)((__uint16_t )(&cmd_slot->prdtl) = ((__uint16_t)(i)) );
3215	}
3216
3217	int
3218	ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
3219	int op, void *page)
3220	{
3221	/* we use the 'real' ahci_port and ahci_softc here, but
3222	* never write to them
3223	*/
3224	struct {
3225	struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */
3226	struct ahci_rfis rfis; /* 1k aligned, 256 bytes */
3227	/* cmd table isn't actually used because of mysteries */
3228	struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */
3229	struct ahci_port *ap;
3230	struct ahci_ccb ccb_buf;
3231	struct ahci_ccb *ccb;
3232	struct ahci_cmd_hdr *hdr_buf;
3233	int pmp_port;
3234	daddr_t poffset;
3235	size_t psize;
3236	} *my = page;
3237	struct ata_fis_h2d *fis;
3238	u_int32_t sector_count;
3239	struct ahci_cmd_hdr *cmd_slot;
3240	int rc;
3241	int timeout;
3242	u_int16_t flags;
3243
3244	if (op == HIB_INIT-1) {
3245	struct device *disk;
3246	struct device *scsibus;
3247	struct ahci_softc *sc;
3248	extern struct cfdriver sd_cd;
3249	struct scsi_link *link;
3250	struct scsibus_softc *bus_sc;
3251	int port;
3252	paddr_t page_phys;
3253	u_int64_t item_phys;
3254	u_int32_t cmd;
3255
3256	my->poffset = blkno;
3257	my->psize = size;
3258
3259	/* map dev to an ahci port */
3260	disk = disk_lookup(&sd_cd, DISKUNIT(dev)(((unsigned)((dev) & 0xff) \| (((dev) & 0xffff0000) >> 8)) / 16));
3261	scsibus = disk->dv_parent;
3262	sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
3263
3264	/* find the scsi_link for the device, which has the port */
3265	port = -1;
3266	bus_sc = (struct scsibus_softc *)scsibus;
3267	SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list)for((link) = ((&bus_sc->sc_link_list)->slh_first); ( link) != ((void *)0); (link) = ((link)->bus_list.sle_next) ) {
3268	if (link->device_softc == disk) {
3269	port = link->target;
3270	if (link->lun > 0)
3271	my->pmp_port = link->lun - 1;
3272	else
3273	my->pmp_port = 0;
3274
3275	break;
3276	}
3277	}
3278	if (port == -1) {
3279	/* don't know where the disk is */
3280	return (EIO5);
3281	}
3282
3283	my->ap = sc->sc_ports[port];
3284
3285	/* we're going to use the first command slot,
3286	* so ensure it's not already in use
3287	*/
3288	if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT6) {
3289	/* this shouldn't happen, we should be idle */
3290	return (EIO5);
3291	}
3292
3293	/* stop the port so we can relocate to the hibernate page */
3294	if (ahci_port_stop(my->ap, 1)) {
3295	return (EIO5);
3296	}
3297	ahci_pwrite(my->ap, AHCI_PREG_SCTL0x2c, 0);
3298
3299	pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)page, &page_phys);
3300
3301	/* Setup RFIS base address */
3302	item_phys = page_phys + ((void *)&my->rfis - page);
3303	ahci_pwrite(my->ap, AHCI_PREG_FBU0x0c,
3304	(u_int32_t)(item_phys >> 32));
3305	ahci_pwrite(my->ap, AHCI_PREG_FB0x08, (u_int32_t)item_phys);
3306
3307	/* Enable FIS reception and activate port. */
3308	cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
3309	cmd \|= AHCI_PREG_CMD_FRE(1<<4) \| AHCI_PREG_CMD_POD(1<<2) \|
3310	AHCI_PREG_CMD_SUD(1<<1);
3311	ahci_pwrite(my->ap, AHCI_PREG_CMD0x18, cmd \|
3312	AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
3313
3314	/* Check whether port activated. */
3315	cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
3316	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
3317	return (EIO5);
3318	}
3319
3320	/* Set up the single CCB */
3321	my->ccb = &my->ccb_buf;
3322	my->ccb->ccb_slot = 0;
3323	my->ccb->ccb_port = my->ap;
3324
3325	/* Setup command list base address */
3326	item_phys = page_phys + ((void *)&my->cmd_hdr - page);
3327	ahci_pwrite(my->ap, AHCI_PREG_CLBU0x04,
3328	(u_int32_t)(item_phys >> 32));
3329	ahci_pwrite(my->ap, AHCI_PREG_CLB0x00, (u_int32_t)item_phys);
3330
3331	my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
3332
3333	/* use existing cmd table - moving to a new one fails */
3334	my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
3335	pmap_extract(pmap_kernel()(&kernel_pmap_store),
3336	(vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table)((void *)(my->ap->ap_dmamem_cmd_table)->adm_kva),
3337	&page_phys);
3338	item_phys = page_phys;
3339	#if 0
3340	/* use cmd table in hibernate page (doesn't work) */
3341	my->ccb->ccb_cmd_table = &my->cmd_table;
3342	item_phys = page_phys + ((void *)&my->cmd_table - page);
3343	#endif
3344	htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys)((__uint64_t )(&my->ccb->ccb_cmd_hdr->ctba) = ( (__uint64_t)(item_phys)));
3345
3346	my->ccb->ccb_xa.fis =
3347	(struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
3348	my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
3349	my->ccb->ccb_xa.tag = 0;
3350
3351	/* Wait for ICC change to complete */
3352	ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((my->ap), (0x18), (0xf0000000), 0, (1));
3353
3354	if (ahci_port_start(my->ap, 0)((my->ap)->ap_sc->sc_port_start((my->ap), (0)))) {
3355	return (EIO5);
3356	}
3357
3358	/* Flush interrupts for port */
3359	ahci_pwrite(my->ap, AHCI_PREG_IS0x10, ahci_pread(my->ap,
3360	AHCI_PREG_IS0x10));
3361	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
3362
3363	ahci_enable_interrupts(my->ap);
3364	return (0);
3365	} else if (op == HIB_DONE-2) {
3366	ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME4);
3367	return (0);
3368	}
3369
3370	if (blkno > my->psize)
3371	return (E2BIG7);
3372	blkno += my->poffset;
3373
3374	/* build fis */
3375	sector_count = size / 512; /* dlg promises this is okay */
3376	my->ccb->ccb_xa.flags = op == HIB_W1 ? ATA_F_WRITE(1<<1) : ATA_F_READ(1<<0);
3377	fis = my->ccb->ccb_xa.fis;
3378	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| my->pmp_port;
3379	fis->lba_low = blkno & 0xff;
3380	fis->lba_mid = (blkno >> 8) & 0xff;
3381	fis->lba_high = (blkno >> 16) & 0xff;
3382
3383	if (sector_count > 0x100 \|\| blkno > 0xfffffff) {
3384	/* Use LBA48 */
3385	fis->command = op == HIB_W1 ? ATA_C_WRITEDMA_EXT0x35 :
3386	ATA_C_READDMA_EXT0x25;
3387	fis->device = ATA_H2D_DEVICE_LBA0x40;
3388	fis->lba_low_exp = (blkno >> 24) & 0xff;
3389	fis->lba_mid_exp = (blkno >> 32) & 0xff;
3390	fis->lba_high_exp = (blkno >> 40) & 0xff;
3391	fis->sector_count = sector_count & 0xff;
3392	fis->sector_count_exp = (sector_count >> 8) & 0xff;
3393	} else {
3394	/* Use LBA */
3395	fis->command = op == HIB_W1 ? ATA_C_WRITEDMA0xca : ATA_C_READDMA0xc8;
3396	fis->device = ATA_H2D_DEVICE_LBA0x40 \| ((blkno >> 24) & 0x0f);
3397	fis->sector_count = sector_count & 0xff;
3398	}
3399
3400	my->ccb->ccb_xa.data = (void *)addr;
3401	my->ccb->ccb_xa.datalen = size;
3402	my->ccb->ccb_xa.pmp_port = my->pmp_port;
3403	my->ccb->ccb_xa.flags \|= ATA_F_POLL(1<<3);
3404
3405	cmd_slot = my->ccb->ccb_cmd_hdr;
3406	flags = 5; /* FIS length (in DWORDs) */
3407	flags \|= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12;
3408
3409	if (op == HIB_W1)
3410	flags \|= AHCI_CMD_LIST_FLAG_W(1<<6);
3411
3412	htolem16(&cmd_slot->flags, flags)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(flags )));
3413
3414	ahci_hibernate_load_prdt(my->ccb);
3415
3416	ahci_hibernate_io_start(my->ap, my->ccb);
3417	timeout = 1000000;
3418	while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN35) {
	Although the value stored to 'rc' is used in the enclosing expression, the value is never actually read from 'rc'
3419	delay(1)(*delay_func)(1);
3420	timeout--;
3421	if (timeout == 0) {
3422	return (EIO5);
3423	}
3424	}
3425
3426	return (0);
3427	}
3428
3429	#endif