/usr/src/sys/dev/ic/ahci.c

Bug Summary

File:	dev/ic/ahci.c
Warning:	line 2275, column 3 Value stored to 'is' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ahci.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/ic/ahci.c

1	/* $OpenBSD: ahci.c,v 1.39 2023/02/03 18:31:16 miod Exp $ */
2
3	/*
4	* Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
5	* Copyright (c) 2010 Conformal Systems LLC <info@conformal.com>
6	* Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org>
7	*
8	* Permission to use, copy, modify, and distribute this software for any
9	* purpose with or without fee is hereby granted, provided that the above
10	* copyright notice and this permission notice appear in all copies.
11	*
12	* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13	* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14	* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15	* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17	* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18	* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19	*/
20
21	#include <sys/param.h>
22	#include <sys/systm.h>
23	#include <sys/buf.h>
24	#include <sys/kernel.h>
25	#include <sys/malloc.h>
26	#include <sys/device.h>
27	#include <sys/queue.h>
28	#include <sys/mutex.h>
29	#include <sys/pool.h>
30
31	#include <machine/bus.h>
32
33	#include <dev/ic/ahcireg.h>
34	#include <dev/ic/ahcivar.h>
35
36	#ifdef AHCI_DEBUG
37	#define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
38	while (0)
39	#define AHCI_D_TIMEOUT 0x00
40	#define AHCI_D_VERBOSE 0x01
41	#define AHCI_D_INTR 0x02
42	#define AHCI_D_XFER 0x08
43	int ahcidebug = AHCI_D_VERBOSE;
44	#else
45	#define DPRINTF(m, f...)
46	#endif
47
48	#ifdef HIBERNATE1
49	#include <uvm/uvm_extern.h>
50	#include <sys/hibernate.h>
51	#include <sys/disk.h>
52	#include <sys/disklabel.h>
53
54	#include <scsi/scsi_all.h>
55	#include <scsi/scsiconf.h>
56
57	void ahci_hibernate_io_start(struct ahci_port *,
58	struct ahci_ccb *);
59	int ahci_hibernate_io_poll(struct ahci_port *,
60	struct ahci_ccb *);
61	void ahci_hibernate_load_prdt(struct ahci_ccb *);
62
63	int ahci_hibernate_io(dev_t dev, daddr_t blkno,
64	vaddr_t addr, size_t size, int wr, void *page);
65	#endif
66
67	struct cfdriver ahci_cd = {
68	NULL((void *)0), "ahci", DV_DULL
69	};
70
71	void ahci_enable_interrupts(struct ahci_port *);
72
73	int ahci_init(struct ahci_softc *);
74	int ahci_port_alloc(struct ahci_softc *, u_int);
75	void ahci_port_detect(struct ahci_softc *, u_int);
76	void ahci_port_free(struct ahci_softc *, u_int);
77	int ahci_port_init(struct ahci_softc *, u_int);
78
79	int ahci_default_port_start(struct ahci_port *, int);
80	int ahci_port_stop(struct ahci_port *, int);
81	int ahci_port_clo(struct ahci_port *);
82	int ahci_port_softreset(struct ahci_port *);
83	void ahci_port_comreset(struct ahci_port *);
84	int ahci_port_portreset(struct ahci_port *, int);
85	void ahci_port_portreset_start(struct ahci_port *);
86	int ahci_port_portreset_poll(struct ahci_port *);
87	void ahci_port_portreset_wait(struct ahci_port *);
88	int ahci_port_portreset_finish(struct ahci_port *, int);
89	int ahci_port_signature(struct ahci_port *);
90	int ahci_pmp_port_softreset(struct ahci_port *, int);
91	int ahci_pmp_port_portreset(struct ahci_port *, int);
92	int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
93
94	int ahci_load_prdt(struct ahci_ccb *);
95	void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
96	u_int32_t, u_int32_t);
97	void ahci_unload_prdt(struct ahci_ccb *);
98
99	int ahci_poll(struct ahci_ccb , int, void ()(void *));
100	void ahci_start(struct ahci_ccb *);
101
102	void ahci_issue_pending_ncq_commands(struct ahci_port *);
103	void ahci_issue_pending_commands(struct ahci_port *, int);
104
105	int ahci_intr(void *);
106	u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
107
108	struct ahci_ccb ahci_get_ccb(struct ahci_port );
109	void ahci_put_ccb(struct ahci_ccb *);
110
111	struct ahci_ccb ahci_get_err_ccb(struct ahci_port );
112	void ahci_put_err_ccb(struct ahci_ccb *);
113
114	struct ahci_ccb ahci_get_pmp_ccb(struct ahci_port );
115	void ahci_put_pmp_ccb(struct ahci_ccb *);
116
117	int ahci_port_read_ncq_error(struct ahci_port , int , int);
118
119	struct ahci_dmamem ahci_dmamem_alloc(struct ahci_softc , size_t);
120	void ahci_dmamem_free(struct ahci_softc *,
121	struct ahci_dmamem *);
122
123	u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
124	void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
125	int ahci_wait_ne(struct ahci_softc *, bus_size_t,
126	u_int32_t, u_int32_t);
127
128	u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
129	void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
130	int ahci_pwait_eq(struct ahci_port *, bus_size_t,
131	u_int32_t, u_int32_t, int);
132	void ahci_flush_tfd(struct ahci_port *ap);
133	u_int32_t ahci_active_mask(struct ahci_port *);
134	int ahci_port_detect_pmp(struct ahci_port *);
135	void ahci_pmp_probe_timeout(void *);
136
137	/* pmp operations */
138	int ahci_pmp_read(struct ahci_port *, int, int,
139	u_int32_t *);
140	int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
141	int ahci_pmp_phy_status(struct ahci_port *, int,
142	u_int32_t *);
143	int ahci_pmp_identify(struct ahci_port , int );
144
145
146	/* Wait for all bits in _b to be cleared */
147	#define ahci_pwait_clr(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), 0, (_n)) \
148	ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
149
150	/* Wait for all bits in _b to be set */
151	#define ahci_pwait_set(_ap, _r, _b, _n)ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n)) \
152	ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
153
154
155
156	/* provide methods for atascsi to call */
157	int ahci_ata_probe(void *, int, int);
158	void ahci_ata_free(void *, int, int);
159	struct ata_xfer * ahci_ata_get_xfer(void *, int);
160	void ahci_ata_put_xfer(struct ata_xfer *);
161	void ahci_ata_cmd(struct ata_xfer *);
162
163	const struct atascsi_methods ahci_atascsi_methods = {
164	ahci_ata_probe,
165	ahci_ata_free,
166	ahci_ata_get_xfer,
167	ahci_ata_put_xfer,
168	ahci_ata_cmd
169	};
170
171	/* ccb completions */
172	void ahci_ata_cmd_done(struct ahci_ccb *);
173	void ahci_pmp_cmd_done(struct ahci_ccb *);
174	void ahci_ata_cmd_timeout(void *);
175	void ahci_empty_done(struct ahci_ccb *);
176
177	int
178	ahci_attach(struct ahci_softc *sc)
179	{
180	struct atascsi_attach_args aaa;
181	u_int32_t pi;
182	int i, j, done;
183
184	if (sc->sc_port_start == NULL((void *)0))
185	sc->sc_port_start = ahci_default_port_start;
186
187	if (ahci_init(sc) != 0) {
188	/* error already printed by ahci_init */
189	goto unmap;
190	}
191
192	printf("\n");
193
194	sc->sc_cap = ahci_read(sc, AHCI_REG_CAP0x000);
195	sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap)((((sc->sc_cap) & 0x1f00)>>8)+1);
196	#ifdef AHCI_DEBUG
197	if (ahcidebug & AHCI_D_VERBOSE) {
198	const char *gen;
199
200	switch (sc->sc_cap & AHCI_REG_CAP_ISS(0xf<<20)) {
201	case AHCI_REG_CAP_ISS_G1(0x1<<20):
202	gen = "1 (1.5Gbps)";
203	break;
204	case AHCI_REG_CAP_ISS_G2(0x2<<20):
205	gen = "2 (3.0Gb/s)";
206	break;
207	case AHCI_REG_CAP_ISS_G3(0x3<<20):
208	gen = "3 (6.0Gb/s)";
209	break;
210	default:
211	gen = "unknown";
212	break;
213	}
214
215	printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
216	DEVNAME(sc)((sc)->sc_dev.dv_xname), sc->sc_cap, AHCI_FMT_CAP"\020" "\040S64A" "\037NCQ" "\036SSNTF" "\035SMPS" "\034SSS" "\033SALP" "\032SAL" "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" "\021FBSS" "\020PMD" "\017SSC" "\016PSC" "\010CCCS" "\007EMS" "\006SXS",
217	AHCI_REG_CAP_NP(sc->sc_cap)(((sc->sc_cap) & 0x1f)+1), sc->sc_ncmds, gen);
218	printf("%s: extended capabilities 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
219	ahci_read(sc, AHCI_REG_CAP20x024), AHCI_FMT_CAP2"\020" "\006DESO" "\005SADM" "\004SDS" "\003APST" "\002NVMP" "\001BOH");
220	}
221	#endif
222
223	pi = ahci_read(sc, AHCI_REG_PI0x00c);
224	DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
225	DEVNAME(sc), pi);
226
227	#ifdef AHCI_COALESCE
228	/* Naive coalescing support - enable for all ports. */
229	if (sc->sc_cap & AHCI_REG_CAP_CCCS(1<<7)) {
230	u_int16_t ccc_timeout = 20;
231	u_int8_t ccc_numcomplete = 12;
232	u_int32_t ccc_ctl;
233
234	/* disable coalescing during reconfiguration. */
235	ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL0x014);
236	ccc_ctl &= ~0x00000001;
237	ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl);
238
239	sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl)(((ccc_ctl) & 0xf8) >> 3);
240	if (pi & sc->sc_ccc_mask) {
241	/* A conflict with the implemented port list? */
242	printf("%s: coalescing interrupt/implemented port list "
243	"conflict, PI: %08x, ccc_mask: %08x\n",
244	DEVNAME(sc)((sc)->sc_dev.dv_xname), pi, sc->sc_ccc_mask);
245	sc->sc_ccc_mask = 0;
246	goto noccc;
247	}
248
249	/* ahci_port_start will enable each port when it starts. */
250	sc->sc_ccc_ports = pi;
251	sc->sc_ccc_ports_cur = 0;
252
253	/* program thresholds and enable overall coalescing. */
254	ccc_ctl &= ~0xffffff00;
255	ccc_ctl \|= (ccc_timeout << 16) \| (ccc_numcomplete << 8);
256	ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl);
257	ahci_write(sc, AHCI_REG_CCC_PORTS0x018, 0);
258	ahci_write(sc, AHCI_REG_CCC_CTL0x014, ccc_ctl \| 1);
259	}
260	noccc:
261	#endif
262	/*
263	* Given that ahci_port_alloc() will grab one CCB for error recovery
264	* in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds
265	* pretend at least 2 command slots for devices without NCQ support.
266	* That way, also at least 1 slot is made available for atascsi(4).
267	*/
268	sc->sc_ncmds = max(2, sc->sc_ncmds);
269	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
270	if (!ISSET(pi, 1 << i)((pi) & (1 << i))) {
271	/* dont allocate stuff if the port isnt implemented */
272	continue;
273	}
274
275	if (ahci_port_alloc(sc, i) == ENOMEM12)
276	goto freeports;
277
278	if (sc->sc_ports[i] != NULL((void *)0))
279	ahci_port_portreset_start(sc->sc_ports[i]);
280	}
281
282	/*
283	* Poll for device detection until all ports report a device, or one
284	* second has elapsed.
285	*/
286	for (i = 0; i < 1000; i++) {
287	done = 1;
288	for (j = 0; j < AHCI_MAX_PORTS32; j++) {
289	if (sc->sc_ports[j] == NULL((void *)0))
290	continue;
291
292	if (ahci_port_portreset_poll(sc->sc_ports[j]))
293	done = 0;
294	}
295
296	if (done)
297	break;
298
299	delay(1000)(*delay_func)(1000);
300	}
301
302	/*
303	* Finish device detection on all ports that initialized.
304	*/
305	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
306	if (sc->sc_ports[i] != NULL((void *)0))
307	ahci_port_detect(sc, i);
308	}
309
310	memset(&aaa, 0, sizeof(aaa))__builtin_memset((&aaa), (0), (sizeof(aaa)));
311	aaa.aaa_cookie = sc;
312	aaa.aaa_methods = &ahci_atascsi_methods;
313	aaa.aaa_minphys = NULL((void *)0);
314	aaa.aaa_nports = AHCI_MAX_PORTS32;
315	aaa.aaa_ncmds = sc->sc_ncmds - 1;
316	if (!(sc->sc_flags & AHCI_F_NO_NCQ(1<<0)) &&
317	sc->sc_ncmds > 2 &&
318	(sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30))) {
319	aaa.aaa_capability \|= ASAA_CAP_NCQ(1 << 0) \| ASAA_CAP_PMP_NCQ(1 << 2);
320	}
321
322	sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
323
324	/* Enable interrupts */
325	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) \| AHCI_REG_GHC_IE(1<<1));
326
327	return 0;
328
329	freeports:
330	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
331	if (sc->sc_ports[i] != NULL((void *)0))
332	ahci_port_free(sc, i);
333	}
334	unmap:
335	/* Disable controller */
336	ahci_write(sc, AHCI_REG_GHC0x004, 0);
337	return 1;
338	}
339
340	int
341	ahci_detach(struct ahci_softc *sc, int flags)
342	{
343	int rv, i;
344
345	if (sc->sc_atascsi != NULL((void *)0)) {
346	rv = atascsi_detach(sc->sc_atascsi, flags);
347	if (rv != 0)
348	return (rv);
349	}
350
351	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
352	if (sc->sc_ports[i] != NULL((void *)0))
353	ahci_port_free(sc, i);
354	}
355
356	return (0);
357	}
358
359	int
360	ahci_activate(struct device *self, int act)
361	{
362	struct ahci_softc sc = (struct ahci_softc )self;
363	int i, rv = 0;
364
365	switch (act) {
366	case DVACT_RESUME4:
367	/* enable ahci (global interrupts disabled) */
368	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31));
369
370	/* restore BIOS initialised parameters */
371	ahci_write(sc, AHCI_REG_CAP0x000, sc->sc_cap);
372
373	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
374	if (sc->sc_ports[i] != NULL((void *)0))
375	ahci_port_init(sc, i);
376	}
377
378	/* Enable interrupts */
379	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31) \| AHCI_REG_GHC_IE(1<<1));
380
381	rv = config_activate_children(self, act);
382	break;
383	case DVACT_POWERDOWN6:
384	rv = config_activate_children(self, act);
385	for (i = 0; i < AHCI_MAX_PORTS32; i++) {
386	if (sc->sc_ports[i] != NULL((void *)0))
387	ahci_port_stop(sc->sc_ports[i], 1);
388	}
389	break;
390	default:
391	rv = config_activate_children(self, act);
392	break;
393	}
394	return (rv);
395	}
396
397	int
398	ahci_init(struct ahci_softc *sc)
399	{
400	u_int32_t reg, cap, pi;
401	const char *revision;
402
403	DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
404	AHCI_FMT_GHC);
405
406	/* save BIOS initialised parameters, enable staggered spin up */
407	cap = ahci_read(sc, AHCI_REG_CAP0x000);
408	cap &= AHCI_REG_CAP_SMPS(1<<28);
409	cap \|= AHCI_REG_CAP_SSS(1<<27);
410	pi = ahci_read(sc, AHCI_REG_PI0x00c);
411
412	if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))(((1<<31)) & (ahci_read(sc, 0x004)))) {
413	/* reset the controller */
414	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0));
415	if (ahci_wait_ne(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_HR(1<<0),
416	AHCI_REG_GHC_HR(1<<0)) != 0) {
417	printf(" unable to reset controller\n");
418	return (1);
419	}
420	}
421
422	/* enable ahci (global interrupts disabled) */
423	ahci_write(sc, AHCI_REG_GHC0x004, AHCI_REG_GHC_AE(1<<31));
424
425	/* restore parameters */
426	ahci_write(sc, AHCI_REG_CAP0x000, cap);
427	ahci_write(sc, AHCI_REG_PI0x00c, pi);
428
429	/* check the revision */
430	reg = ahci_read(sc, AHCI_REG_VS0x010);
431	switch (reg) {
432	case AHCI_REG_VS_0_950x00000905:
433	revision = "0.95";
434	break;
435	case AHCI_REG_VS_1_00x00010000:
436	revision = "1.0";
437	break;
438	case AHCI_REG_VS_1_10x00010100:
439	revision = "1.1";
440	break;
441	case AHCI_REG_VS_1_20x00010200:
442	revision = "1.2";
443	break;
444	case AHCI_REG_VS_1_30x00010300:
445	revision = "1.3";
446	break;
447	case AHCI_REG_VS_1_3_10x00010301:
448	revision = "1.3.1";
449	break;
450
451	default:
452	printf(" unsupported AHCI revision 0x%08x\n", reg);
453	return (1);
454	}
455
456	printf(" AHCI %s", revision);
457
458	return (0);
459	}
460
461	void
462	ahci_enable_interrupts(struct ahci_port *ap)
463	{
464	ahci_pwrite(ap, AHCI_PREG_IE0x14, AHCI_PREG_IE_TFEE(1<<30) \| AHCI_PREG_IE_HBFE(1<<29) \|
465	AHCI_PREG_IE_IFE(1<<27) \| AHCI_PREG_IE_OFE(1<<24) \| AHCI_PREG_IE_DPE(1<<5) \|
466	AHCI_PREG_IE_UFE(1<<4) \|
467	((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29)) ? AHCI_PREG_IE_IPME(1<<23) : 0) \|
468	#ifdef AHCI_COALESCE
469	((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
470	(AHCI_PREG_IE_SDBE(1<<3) \| AHCI_PREG_IE_DHRE(1<<0)))
471	#else
472	AHCI_PREG_IE_SDBE(1<<3) \| AHCI_PREG_IE_DHRE(1<<0)
473	#endif
474	);
475	}
476
477	int
478	ahci_port_alloc(struct ahci_softc *sc, u_int port)
479	{
480	struct ahci_port *ap;
481	struct ahci_ccb *ccb;
482	u_int64_t dva;
483	u_int32_t cmd;
484	struct ahci_cmd_hdr *hdr;
485	struct ahci_cmd_table *table;
486	int i, rc = ENOMEM12;
487
488	ap = malloc(sizeof(*ap), M_DEVBUF2, M_NOWAIT0x0002 \| M_ZERO0x0008);
489	if (ap == NULL((void *)0)) {
490	printf("%s: unable to allocate memory for port %d\n",
491	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
492	goto reterr;
493	}
494	ap->ap_err_scratch = dma_alloc(DEV_BSIZE(1 << 9), PR_NOWAIT0x0002 \| PR_ZERO0x0008);
495	if (ap->ap_err_scratch == NULL((void *)0)) {
496	printf("%s: unable to allocate DMA scratch buf for port %d\n",
497	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
498	free(ap, M_DEVBUF2, sizeof(*ap));
499	goto reterr;
500	}
501
502	#ifdef AHCI_DEBUG
503	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
504	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
505	#endif
506	ap->ap_port = port;
507	sc->sc_ports[port] = ap;
508
509	if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
510	AHCI_PORT_REGION(port)(0x100 + ((port) * 0x80)), AHCI_PORT_SIZE0x80, &ap->ap_ioh) != 0) {
511	printf("%s: unable to create register window for port %d\n",
512	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
513	goto freeport;
514	}
515
516	ap->ap_sc = sc;
517	#ifdef AHCI_COALESCE
518	ap->ap_num = port;
519	#endif
520	TAILQ_INIT(&ap->ap_ccb_free)do { (&ap->ap_ccb_free)->tqh_first = ((void *)0); ( &ap->ap_ccb_free)->tqh_last = &(&ap->ap_ccb_free )->tqh_first; } while (0);
521	TAILQ_INIT(&ap->ap_ccb_pending)do { (&ap->ap_ccb_pending)->tqh_first = ((void *)0) ; (&ap->ap_ccb_pending)->tqh_last = &(&ap-> ap_ccb_pending)->tqh_first; } while (0);
522	mtx_init(&ap->ap_ccb_mtx, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&ap-> ap_ccb_mtx), ((((0x3)) > 0x0 && ((0x3)) < 0x9) ? 0x9 : ((0x3)))); } while (0);
523
524	/* Disable port interrupts */
525	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
526
527	/* Sec 10.1.2 - deinitialise port if it is already running */
528	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
529	if (ISSET(cmd, (AHCI_PREG_CMD_ST \| AHCI_PREG_CMD_CR \|((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14))))
530	AHCI_PREG_CMD_FRE \| AHCI_PREG_CMD_FR))((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14)))) \|\|
531	ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) {
532	int r;
533
534	r = ahci_port_stop(ap, 1);
535	if (r) {
536	printf("%s: unable to disable %s, ignoring port %d\n",
537	DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port);
538	rc = ENXIO6;
539	goto freeport;
540	}
541
542	/* Write DET to zero */
543	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
544	}
545
546	/* Allocate RFIS */
547	ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
548	if (ap->ap_dmamem_rfis == NULL((void *)0))
549	goto nomem;
550
551	/* Setup RFIS base address */
552	ap->ap_rfis = (struct ahci_rfis ) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void )(ap->ap_dmamem_rfis)->adm_kva);
553	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0 ].ds_addr);
554	ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32));
555	ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva);
556
557	/* Enable FIS reception and activate port. */
558	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
559	cmd \|= AHCI_PREG_CMD_FRE(1<<4) \| AHCI_PREG_CMD_POD(1<<2) \| AHCI_PREG_CMD_SUD(1<<1);
560	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd \| AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
561
562	/* Check whether port activated. Skip it if not. */
563	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
564	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
565	rc = ENXIO6;
566	goto freeport;
567	}
568
569	/* Allocate a CCB for each command slot */
570	ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
571	M_DEVBUF2, M_NOWAIT0x0002 \| M_ZERO0x0008);
572	if (ap->ap_ccbs == NULL((void *)0)) {
573	printf("%s: unable to allocate command list for port %d\n",
574	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
575	goto freeport;
576	}
577
578	/* Command List Structures and Command Tables */
579	ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
580	sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
581	ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
582	sc->sc_ncmds * sizeof(struct ahci_cmd_table));
583	if (ap->ap_dmamem_cmd_table == NULL((void )0) \|\| ap->ap_dmamem_cmd_list == NULL((void )0)) {
584	nomem:
585	printf("%s: unable to allocate DMA memory for port %d\n",
586	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
587	goto freeport;
588	}
589
590	/* Setup command list base address */
591	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs [0].ds_addr);
592	ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32));
593	ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva);
594
595	/* Split CCB allocation into CCBs and assign to command header/table */
596	hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list)((void *)(ap->ap_dmamem_cmd_list)->adm_kva);
597	table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table)((void *)(ap->ap_dmamem_cmd_table)->adm_kva);
598	for (i = 0; i < sc->sc_ncmds; i++) {
599	ccb = &ap->ap_ccbs[i];
600
601	if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,((sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 \| 0x0002), (&ccb->ccb_dmamap))
602	(4 * 1024 * 1024), 0, BUS_DMA_NOWAIT \| BUS_DMA_ALLOCNOW,((sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 \| 0x0002), (&ccb->ccb_dmamap))
603	&ccb->ccb_dmamap)((sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), ((64 1024)), (24), ((4 * 1024 * 1024)), (0), (0x0001 \| 0x0002), (&ccb->ccb_dmamap)) != 0) {
604	printf("%s: unable to create dmamap for port %d "
605	"ccb %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port, i);
606	goto freeport;
607	}
608
609	ccb->ccb_slot = i;
610	ccb->ccb_port = ap;
611	ccb->ccb_cmd_hdr = &hdr[i];
612	ccb->ccb_cmd_table = &table[i];
613	htolem64(&ccb->ccb_cmd_hdr->ctba,((__uint64_t )(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table ))))
614	AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +((__uint64_t )(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table ))))
615	ccb->ccb_slot * sizeof(struct ahci_cmd_table))((__uint64_t )(&ccb->ccb_cmd_hdr->ctba) = ((__uint64_t )(((u_int64_t)(ap->ap_dmamem_cmd_table)->adm_map->dm_segs [0].ds_addr) + ccb->ccb_slot * sizeof(struct ahci_cmd_table ))));
616
617	ccb->ccb_xa.fis =
618	(struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
619	ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
620	ccb->ccb_xa.tag = i;
621
622	ccb->ccb_xa.state = ATA_S_COMPLETE2;
623	ahci_put_ccb(ccb);
624	}
625
626	/* grab a ccb for use during error recovery */
627	ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
628	TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry)do { if (((ap->ap_ccb_err)->ccb_entry.tqe_next) != ((void )0)) (ap->ap_ccb_err)->ccb_entry.tqe_next->ccb_entry .tqe_prev = (ap->ap_ccb_err)->ccb_entry.tqe_prev; else ( &ap->ap_ccb_free)->tqh_last = (ap->ap_ccb_err)-> ccb_entry.tqe_prev; (ap->ap_ccb_err)->ccb_entry.tqe_prev = (ap->ap_ccb_err)->ccb_entry.tqe_next; ((ap->ap_ccb_err )->ccb_entry.tqe_prev) = ((void )-1); ((ap->ap_ccb_err )->ccb_entry.tqe_next) = ((void )-1); } while (0);
629	ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE2;
630
631	/* Wait for ICC change to complete */
632	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1));
633	rc = 0;
634
635	freeport:
636	if (rc != 0)
637	ahci_port_free(sc, port);
638	reterr:
639	return (rc);
640	}
641
642	void
643	ahci_port_detect(struct ahci_softc *sc, u_int port)
644	{
645	struct ahci_port *ap;
646	const char *speed;
647	int rc;
648
649	ap = sc->sc_ports[port];
650
651	rc = ahci_port_portreset_finish(ap, 1);
652	switch (rc) {
653	case ENODEV19:
654	switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
655	case AHCI_PREG_SSTS_DET_DEV_NE0x1:
656	printf("%s: device not communicating on port %d\n",
657	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
658	break;
659	case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4:
660	printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
661	port);
662	break;
663	default:
664	DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
665	"on port %d\n", DEVNAME(sc), port);
666	break;
667	}
668	goto freeport;
669
670	case EBUSY16:
671	printf("%s: device on port %d didn't come ready, "
672	"TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port,
673	ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
674
675	/* Try a soft reset to clear busy */
676	rc = ahci_port_softreset(ap);
677	if (rc) {
678	printf("%s: unable to communicate "
679	"with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
680	goto freeport;
681	}
682	break;
683
684	default:
685	break;
686	}
687
688	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
689	DEVNAME(sc), port, rc);
690
691	/* Read current link speed */
692	switch(ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_SPD0xf0) {
693	case AHCI_PREG_SSTS_SPD_GEN10x10:
694	speed = "1.5Gb/s";
695	break;
696	case AHCI_PREG_SSTS_SPD_GEN20x20:
697	speed = "3.0Gb/s";
698	break;
699	case AHCI_PREG_SSTS_SPD_GEN30x30:
700	speed = "6.0Gb/s";
701	break;
702	default:
703	speed = NULL((void *)0);
704	break;
705	}
706	if (speed != NULL((void *)0))
707	printf("%s: port %d: %s\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), port, speed);
708
709	/* Enable command transfers on port */
710	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
711	printf("%s: failed to start command DMA on port %d, "
712	"disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
713	rc = ENXIO6; /* couldn't start port */
714	}
715
716	/* Flush interrupts for port */
717	ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
718	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
719
720	ahci_enable_interrupts(ap);
721	freeport:
722	if (rc != 0)
723	ahci_port_free(sc, port);
724	}
725
726	void
727	ahci_port_free(struct ahci_softc *sc, u_int port)
728	{
729	struct ahci_port *ap = sc->sc_ports[port];
730	struct ahci_ccb *ccb;
731
732	/* Ensure port is disabled and its interrupts are flushed */
733	if (ap->ap_sc) {
734	ahci_pwrite(ap, AHCI_PREG_CMD0x18, 0);
735	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
736	ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
737	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
738	}
739
740	if (ap->ap_ccb_err)
741	ahci_put_ccb(ap->ap_ccb_err);
742
743	if (ap->ap_ccbs) {
744	while ((ccb = ahci_get_ccb(ap)) != NULL((void *)0))
745	bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (ccb ->ccb_dmamap));
746	free(ap->ap_ccbs, M_DEVBUF2, sc->sc_ncmds * sizeof(*ccb));
747	}
748
749	if (ap->ap_dmamem_cmd_list)
750	ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
751	if (ap->ap_dmamem_rfis)
752	ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
753	if (ap->ap_dmamem_cmd_table)
754	ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
755	if (ap->ap_err_scratch)
756	dma_free(ap->ap_err_scratch, DEV_BSIZE(1 << 9));
757
758	/* bus_space(9) says we dont free the subregions handle */
759
760	free(ap, M_DEVBUF2, sizeof(*ap));
761	sc->sc_ports[port] = NULL((void *)0);
762	}
763
764	int
765	ahci_port_init(struct ahci_softc *sc, u_int port)
766	{
767	struct ahci_port *ap;
768	u_int64_t dva;
769	u_int32_t cmd;
770	int rc = ENOMEM12;
771
772	ap = sc->sc_ports[port];
773	#ifdef AHCI_DEBUG
774	snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
775	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
776	#endif
777
778	/* Disable port interrupts */
779	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
780
781	/* Sec 10.1.2 - deinitialise port if it is already running */
782	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
783	if (ISSET(cmd, (AHCI_PREG_CMD_ST \| AHCI_PREG_CMD_CR \|((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14))))
784	AHCI_PREG_CMD_FRE \| AHCI_PREG_CMD_FR))((cmd) & (((1<<0) \| (1<<15) \| (1<<4) \| ( 1<<14)))) \|\|
785	ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)((ahci_pread(ap, 0x2c)) & (0xf))) {
786	int r;
787
788	r = ahci_port_stop(ap, 1);
789	if (r) {
790	printf("%s: unable to disable %s, ignoring port %d\n",
791	DEVNAME(sc)((sc)->sc_dev.dv_xname), r == 2 ? "CR" : "FR", port);
792	rc = ENXIO6;
793	goto reterr;
794	}
795
796	/* Write DET to zero */
797	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
798	}
799
800	/* Setup RFIS base address */
801	ap->ap_rfis = (struct ahci_rfis ) AHCI_DMA_KVA(ap->ap_dmamem_rfis)((void )(ap->ap_dmamem_rfis)->adm_kva);
802	dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis)((u_int64_t)(ap->ap_dmamem_rfis)->adm_map->dm_segs[0 ].ds_addr);
803	ahci_pwrite(ap, AHCI_PREG_FBU0x0c, (u_int32_t)(dva >> 32));
804	ahci_pwrite(ap, AHCI_PREG_FB0x08, (u_int32_t)dva);
805
806	/* Enable FIS reception and activate port. */
807	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
808	cmd \|= AHCI_PREG_CMD_FRE(1<<4) \| AHCI_PREG_CMD_POD(1<<2) \| AHCI_PREG_CMD_SUD(1<<1);
809	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd \| AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
810
811	/* Check whether port activated. Skip it if not. */
812	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
813	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
814	rc = ENXIO6;
815	goto reterr;
816	}
817
818	/* Setup command list base address */
819	dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list)((u_int64_t)(ap->ap_dmamem_cmd_list)->adm_map->dm_segs [0].ds_addr);
820	ahci_pwrite(ap, AHCI_PREG_CLBU0x04, (u_int32_t)(dva >> 32));
821	ahci_pwrite(ap, AHCI_PREG_CLB0x00, (u_int32_t)dva);
822
823	/* Wait for ICC change to complete */
824	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((ap), (0x18), (0xf0000000), 0, (1));
825
826	/* Reset port */
827	rc = ahci_port_portreset(ap, 1);
828	switch (rc) {
829	case ENODEV19:
830	switch (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
831	case AHCI_PREG_SSTS_DET_DEV_NE0x1:
832	printf("%s: device not communicating on port %d\n",
833	DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
834	break;
835	case AHCI_PREG_SSTS_DET_PHYOFFLINE0x4:
836	printf("%s: PHY offline on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname),
837	port);
838	break;
839	default:
840	DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
841	"on port %d\n", DEVNAME(sc), port);
842	break;
843	}
844	goto reterr;
845
846	case EBUSY16:
847	printf("%s: device on port %d didn't come ready, "
848	"TFD: 0x%b\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port,
849	ahci_pread(ap, AHCI_PREG_TFD0x20), AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
850
851	/* Try a soft reset to clear busy */
852	rc = ahci_port_softreset(ap);
853	if (rc) {
854	printf("%s: unable to communicate "
855	"with device on port %d\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
856	goto reterr;
857	}
858	break;
859
860	default:
861	break;
862	}
863	DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
864	DEVNAME(sc), port);
865
866	if (ap->ap_pmp_ports > 0) {
867	int p;
868
869	for (p = 0; p < ap->ap_pmp_ports; p++) {
870	int sig;
871
872	/* might need to do a portreset first here? */
873
874	/* softreset the port */
875	if (ahci_pmp_port_softreset(ap, p)) {
876	printf("%s.%d: unable to probe PMP port due to"
877	" softreset failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p);
878	continue;
879	}
880
881	sig = ahci_port_signature(ap);
882	printf("%s.%d: port signature returned %d\n",
883	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), p, sig);
884	}
885	}
886
887	/* Enable command transfers on port */
888	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
889	printf("%s: failed to start command DMA on port %d, "
890	"disabling\n", DEVNAME(sc)((sc)->sc_dev.dv_xname), port);
891	rc = ENXIO6; /* couldn't start port */
892	}
893
894	/* Flush interrupts for port */
895	ahci_pwrite(ap, AHCI_PREG_IS0x10, ahci_pread(ap, AHCI_PREG_IS0x10));
896	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
897
898	ahci_enable_interrupts(ap);
899
900	reterr:
901	return (rc);
902	}
903
904	int
905	ahci_default_port_start(struct ahci_port *ap, int fre_only)
906	{
907	u_int32_t r;
908
909	/* Turn on FRE (and ST) */
910	r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
911	r \|= AHCI_PREG_CMD_FRE(1<<4);
912	if (!fre_only)
913	r \|= AHCI_PREG_CMD_ST(1<<0);
914	ahci_pwrite(ap, AHCI_PREG_CMD0x18, r);
915
916	#ifdef AHCI_COALESCE
917	/* (Re-)enable coalescing on the port. */
918	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
919	ap->ap_sc->sc_ccc_ports_cur \|= (1 << ap->ap_num);
920	ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018,
921	ap->ap_sc->sc_ccc_ports_cur);
922	}
923	#endif
924
925	return (0);
926	}
927
928	int
929	ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
930	{
931	u_int32_t r;
932
933	#ifdef AHCI_COALESCE
934	/* Disable coalescing on the port while it is stopped. */
935	if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
936	ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
937	ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS0x018,
938	ap->ap_sc->sc_ccc_ports_cur);
939	}
940	#endif
941
942	/* Turn off ST (and FRE) */
943	r = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
944	r &= ~AHCI_PREG_CMD_ST(1<<0);
945	if (stop_fis_rx)
946	r &= ~AHCI_PREG_CMD_FRE(1<<4);
947	ahci_pwrite(ap, AHCI_PREG_CMD0x18, r);
948
949	/* Wait for CR to go off */
950	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1)ahci_pwait_eq((ap), (0x18), ((1<<15)), 0, (1)))
951	return (1);
952
953	/* Wait for FR to go off */
954	if (stop_fis_rx &&
955	ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1)ahci_pwait_eq((ap), (0x18), ((1<<14)), 0, (1)))
956	return (2);
957
958	return (0);
959	}
960
961	/* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
962	int
963	ahci_port_clo(struct ahci_port *ap)
964	{
965	struct ahci_softc *sc = ap->ap_sc;
966	u_int32_t cmd;
967
968	/* Only attempt CLO if supported by controller */
969	if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(sc, 0x000)) & ((1<<24))))
970	return (1);
971
972	/* Issue CLO */
973	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
974	#ifdef DIAGNOSTIC1
975	if (ISSET(cmd, AHCI_PREG_CMD_ST)((cmd) & ((1<<0))))
976	printf("%s: CLO requested while port running\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
977	#endif
978	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd \| AHCI_PREG_CMD_CLO(1<<3));
979
980	/* Wait for completion */
981	if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)ahci_pwait_eq((ap), (0x18), ((1<<3)), 0, (1))) {
982	printf("%s: CLO did not complete\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
983	return (1);
984	}
985
986	return (0);
987	}
988
989	/* AHCI soft reset, Section 10.4.1 */
990	int
991	ahci_port_softreset(struct ahci_port *ap)
992	{
993	struct ahci_ccb ccb = NULL((void )0);
994	struct ahci_cmd_hdr *cmd_slot;
995	u_int8_t *fis;
996	int s, rc = EIO5, oldstate;
997	u_int32_t cmd;
998
999	DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
1000
1001	s = splbio()splraise(0x3);
1002	oldstate = ap->ap_state;
1003	ap->ap_state = AP_S_ERROR_RECOVERY3;
1004
1005	/* Save previous command register state */
1006	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1007
1008	/* Idle port */
1009	if (ahci_port_stop(ap, 0)) {
1010	printf("%s: failed to stop port, cannot softreset\n",
1011	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1012	goto err;
1013	}
1014
1015	/* Request CLO if device appears hung */
1016	if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY \|((ahci_pread(ap, 0x20)) & ((1<<7) \| (1<<3)))
1017	AHCI_PREG_TFD_STS_DRQ)((ahci_pread(ap, 0x20)) & ((1<<7) \| (1<<3))))
1018	ahci_port_clo(ap);
1019
1020	/* Clear port errors to permit TFD transfer */
1021	ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30));
1022
1023	/* Restart port */
1024	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
1025	printf("%s: failed to start port, cannot softreset\n",
1026	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1027	goto err;
1028	}
1029
1030	/* Check whether CLO worked */
1031	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))
1032	AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))) {
1033	printf("%s: CLO %s, need port reset\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1034	ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)((ahci_read(ap->ap_sc, 0x000)) & ((1<<24)))
1035	? "failed" : "unsupported");
1036	rc = EBUSY16;
1037	goto err;
1038	}
1039
1040	/* Prep first D2H command with SRST feature & clear busy/reset flags */
1041	ccb = ahci_get_err_ccb(ap);
1042	cmd_slot = ccb->ccb_cmd_hdr;
1043	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1044
1045	fis = ccb->ccb_cmd_table->cfis;
1046	fis[0] = ATA_FIS_TYPE_H2D0x27;
1047	fis[15] = ATA_FIS_CONTROL_SRST0x04;
1048
1049	cmd_slot->prdtl = 0;
1050	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (1<<6))))
1051	AHCI_CMD_LIST_FLAG_C \| AHCI_CMD_LIST_FLAG_R \|((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (1<<6))))
1052	AHCI_CMD_LIST_FLAG_W)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (1<<6))));
1053
1054	ccb->ccb_xa.state = ATA_S_PENDING1;
1055	if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0)
1056	goto err;
1057
1058	/* Prep second D2H command to read status and complete reset sequence */
1059	fis[0] = ATA_FIS_TYPE_H2D0x27;
1060	fis[15] = 0;
1061
1062	cmd_slot->prdtl = 0;
1063	htolem16(&cmd_slot->flags, 5 \| AHCI_CMD_LIST_FLAG_W)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<6))));
1064
1065	ccb->ccb_xa.state = ATA_S_PENDING1;
1066	if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0)
1067	goto err;
1068
1069	if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY \|ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (1))
1070	AHCI_PREG_TFD_STS_DRQ \| AHCI_PREG_TFD_STS_ERR, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (1))) {
1071	printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1072	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ahci_pread(ap, AHCI_PREG_TFD0x20),
1073	AHCI_PFMT_TFD_STS"\20" "\010BSY" "\004DRQ" "\001ERR");
1074	rc = EBUSY16;
1075	goto err;
1076	}
1077
1078	rc = 0;
1079	err:
1080	if (ccb != NULL((void *)0)) {
1081	/* Abort our command, if it failed, by stopping command DMA. */
1082	if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) {
1083	printf("%s: stopping the port, softreset slot %d was "
1084	"still active.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot);
1085	ahci_port_stop(ap, 0);
1086	}
1087	ccb->ccb_xa.state = ATA_S_ERROR3;
1088	ahci_put_err_ccb(ccb);
1089	}
1090
1091	/* Restore saved CMD register state */
1092	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1093	ap->ap_state = oldstate;
1094
1095	splx(s)spllower(s);
1096
1097	return (rc);
1098	}
1099
1100	int
1101	ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
1102	{
1103	struct ahci_ccb ccb = NULL((void )0);
1104	u_int32_t data;
1105	int count;
1106	int rc;
1107	int s;
1108	struct ahci_cmd_hdr *cmd_slot;
1109	u_int8_t *fis;
1110
1111	s = splbio()splraise(0x3);
1112	/* ignore spurious IFS errors while resetting */
1113	DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
1114	ap->ap_pmp_ignore_ifs = 1;
1115
1116	count = 2;
1117	rc = 0;
1118	do {
1119	if (ccb != NULL((void *)0)) {
1120	ahci_put_pmp_ccb(ccb);
1121	ccb = NULL((void *)0);
1122	}
1123
1124	if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1125	printf("%s.%d: unable to clear PHY status\n",
1126	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1127	}
1128	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1129	/* maybe don't do this on the first loop: */
1130	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1131	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1132
1133	/* send first softreset FIS */
1134	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1135	cmd_slot = ccb->ccb_cmd_hdr;
1136	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1137
1138	fis = ccb->ccb_cmd_table->cfis;
1139	fis[0] = ATA_FIS_TYPE_H2D0x27;
1140	fis[1] = pmp_port;
1141	fis[15] = ATA_FIS_CONTROL_SRST0x04 \| ATA_FIS_CONTROL_4BIT0x08;
1142
1143	cmd_slot->prdtl = 0;
1144	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (pmp_port << 12))))
1145	AHCI_CMD_LIST_FLAG_C \| AHCI_CMD_LIST_FLAG_R \|((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (pmp_port << 12))))
1146	(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| (pmp_port << 12))));
1147
1148	ccb->ccb_xa.state = ATA_S_PENDING1;
1149
1150	DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
1151	PORTNAME(ap), pmp_port);
1152	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1153	printf("%s.%d: PMP port softreset cmd failed\n",
1154	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1155	rc = EBUSY16;
1156	if (count > 0) {
1157	/* probably delay a while to allow
1158	* it to settle down?
1159	*/
1160	}
1161	continue;
1162	}
1163
1164	/* send signature FIS */
1165	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1166	fis[0] = ATA_FIS_TYPE_H2D0x27;
1167	fis[1] = pmp_port;
1168	fis[15] = ATA_FIS_CONTROL_4BIT0x08;
1169
1170	cmd_slot->prdtl = 0;
1171	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))))
1172	(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))));
1173
1174	DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
1175	PORTNAME(ap), pmp_port);
1176	ccb->ccb_xa.state = ATA_S_PENDING1;
1177	if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1178	DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
1179	"failed\n", PORTNAME(ap), pmp_port);
1180	rc = EBUSY16;
1181	if (count > 0) {
1182	/* sleep a while? */
1183	}
1184	continue;
1185	}
1186
1187	fis[15] = 0;
1188	break;
1189	} while (count--);
1190
1191	if (ccb != NULL((void *)0)) {
1192	ahci_put_pmp_ccb(ccb);
1193	ccb = NULL((void *)0);
1194	}
1195
1196	/* clean up a bit */
1197	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1198	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1199	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1200	ap->ap_pmp_ignore_ifs = 0;
1201	DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
1202	splx(s)spllower(s);
1203
1204	return (rc);
1205	}
1206
1207	int
1208	ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
1209	{
1210	int sig;
1211
1212	ap->ap_state = AP_S_PMP_PORT_PROBE2;
1213
1214	DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
1215	pmp_port);
1216	if (ahci_pmp_port_portreset(ap, pmp_port)) {
1217	printf("%s.%d: unable to probe PMP port; portreset failed\n",
1218	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1219	ap->ap_state = AP_S_NORMAL0;
1220	return (ATA_PORT_T_NONE0);
1221	}
1222
1223	if (ahci_pmp_port_softreset(ap, pmp_port)) {
1224	printf("%s.%d: unable to probe PMP port due to softreset "
1225	"failure\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1226	ap->ap_state = AP_S_NORMAL0;
1227	return (ATA_PORT_T_NONE0);
1228	}
1229
1230	sig = ahci_port_signature(ap);
1231	DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
1232	PORTNAME(ap), pmp_port, sig);
1233	ap->ap_state = AP_S_NORMAL0;
1234	return (sig);
1235	}
1236
1237
1238	void
1239	ahci_flush_tfd(struct ahci_port *ap)
1240	{
1241	u_int32_t r;
1242
1243	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1244	if (r & AHCI_PREG_SERR_DIAG_X(1<<10))
1245	ahci_pwrite(ap, AHCI_PREG_SERR0x30, AHCI_PREG_SERR_DIAG_X(1<<10));
1246	}
1247
1248	u_int32_t
1249	ahci_active_mask(struct ahci_port *ap)
1250	{
1251	u_int32_t mask;
1252
1253	mask = ahci_pread(ap, AHCI_PREG_CI0x38);
1254	if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ(1<<30))
1255	mask \|= ahci_pread(ap, AHCI_PREG_SACT0x34);
1256	return mask;
1257	}
1258
1259	void
1260	ahci_pmp_probe_timeout(void *cookie)
1261	{
1262	struct ahci_ccb *ccb = cookie;
1263	struct ahci_port *ap = ccb->ccb_port;
1264	u_int32_t mask;
1265
1266	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
1267	switch (ccb->ccb_xa.state) {
1268	case ATA_S_PENDING1:
1269	TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = ( ccb)->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = ( ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
1270	ccb->ccb_xa.state = ATA_S_TIMEOUT4;
1271	break;
1272
1273	case ATA_S_ONCHIP5:
1274	case ATA_S_ERROR3: /* currently mostly here for the ATI SBx00 quirk */
1275	/* clear the command on-chip */
1276	KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&((ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0" ))
1277	ap->ap_sactive == 0)((ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1277, "ap->ap_active == (1 << ccb->ccb_slot) && ap->ap_sactive == 0" ));
1278	ahci_port_stop(ap, 0);
1279	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
1280
1281	if (ahci_active_mask(ap) != 0) {
1282	ahci_port_stop(ap, 0);
1283	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
1284	mask = ahci_active_mask(ap);
1285	if (mask != 0) {
1286	printf("%s: ahci_pmp_probe_timeout: failed to "
1287	"clear active cmds: %08x\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1288	mask);
1289	}
1290	}
1291
1292	ccb->ccb_xa.state = ATA_S_TIMEOUT4;
1293	ap->ap_active &= ~(1 << ccb->ccb_slot);
1294	KASSERT(ap->ap_active_cnt > 0)((ap->ap_active_cnt > 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1294, "ap->ap_active_cnt > 0" ));
1295	--ap->ap_active_cnt;
1296	DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, "
1297	"active_cnt %d\n", PORTNAME(ap), ccb->ccb_slot,
1298	ap->ap_active, ap->ap_active_cnt);
1299	break;
1300
1301	default:
1302	panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
1303	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state);
1304	}
1305	}
1306
1307	int
1308	ahci_port_signature(struct ahci_port *ap)
1309	{
1310	u_int32_t sig;
1311
1312	sig = ahci_pread(ap, AHCI_PREG_SIG0x24);
1313	if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI0xeb140101 & 0xffff0000))
1314	return (ATA_PORT_T_ATAPI2);
1315	else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER0x96690101 &
1316	0xffff0000))
1317	return (ATA_PORT_T_PM3);
1318	else
1319	return (ATA_PORT_T_DISK1);
1320	}
1321
1322	int
1323	ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
1324	{
1325	u_int32_t cmd, data;
1326	int loop;
1327	int rc = 1;
1328	int s;
1329
1330	s = splbio()splraise(0x3);
1331	DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
1332	pmp_port);
1333
1334	/* Save previous command register state */
1335	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1336
1337	/* turn off power management and disable the PHY */
1338	data = AHCI_PREG_SCTL_IPM_DISABLED0x300;
1339	/* maybe add AHCI_PREG_SCTL_DET_DISABLE */
1340	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1))
1341	goto err;
1342	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1343	goto err;
1344	delay(10000)(*delay_func)(10000);
1345
1346	/* start COMRESET */
1347	data = AHCI_PREG_SCTL_IPM_DISABLED0x300 \| AHCI_PREG_SCTL_DET_INIT0x1;
1348	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1349	DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
1350	pmp_port);
1351	data \|= AHCI_PREG_SCTL_SPD_GEN10x10;
1352	} else
1353	data \|= AHCI_PREG_SCTL_SPD_ANY0x00;
1354
1355	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1356	goto err;
1357
1358	/* give it a while to settle down */
1359	delay(100000)(*delay_func)(100000);
1360
1361	if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1362	printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1363	pmp_port);
1364	}
1365
1366	/* start trying to negotiate */
1367	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1368	data = AHCI_PREG_SCTL_IPM_DISABLED0x300 \| AHCI_PREG_SCTL_DET_NONE0x0;
1369	if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL2, data))
1370	goto err;
1371
1372	/* give it a while to detect */
1373	for (loop = 3; loop; --loop) {
1374	if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data))
1375	goto err;
1376	if (data & AHCI_PREG_SSTS_DET0xf)
1377	break;
1378	delay(100000)(*delay_func)(100000);
1379	}
1380	if (loop == 0) {
1381	printf("%s.%d: port is unplugged\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), pmp_port);
1382	goto err;
1383	}
1384
1385	/* give it even longer to fully negotiate */
1386	for (loop = 30; loop; --loop) {
1387	if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS0, &data))
1388	goto err;
1389	if ((data & AHCI_PREG_SSTS_DET0xf) == AHCI_PREG_SSTS_DET_DEV0x3)
1390	break;
1391	delay(100000)(*delay_func)(100000);
1392	}
1393
1394	if (loop == 0) {
1395	printf("%s.%d: device is not negotiating\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1396	pmp_port);
1397	goto err;
1398	}
1399
1400	/* device detected */
1401	DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
1402	pmp_port);
1403
1404	/* clean up a bit */
1405	delay(100000)(*delay_func)(100000);
1406	ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR1, -1);
1407	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1408	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
1409
1410	rc = 0;
1411	err:
1412	/* Restore preserved port state */
1413	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1414	splx(s)spllower(s);
1415	return (rc);
1416	}
1417
1418	/* AHCI port reset, Section 10.4.2 */
1419
1420	void
1421	ahci_port_comreset(struct ahci_port *ap)
1422	{
1423	u_int32_t r;
1424
1425	r = AHCI_PREG_SCTL_IPM_DISABLED0x300 \| AHCI_PREG_SCTL_DET_INIT0x1;
1426	if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1427	DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
1428	r \|= AHCI_PREG_SCTL_SPD_GEN10x10;
1429	} else
1430	r \|= AHCI_PREG_SCTL_SPD_ANY0x00;
1431	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r);
1432	delay(10000)(delay_func)(10000); / wait at least 1ms for COMRESET to be sent */
1433	r &= ~AHCI_PREG_SCTL_DET_INIT0x1;
1434	r \|= AHCI_PREG_SCTL_DET_NONE0x0;
1435	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, r);
1436	delay(10000)(*delay_func)(10000);
1437	}
1438
1439	void
1440	ahci_port_portreset_start(struct ahci_port *ap)
1441	{
1442	int s;
1443
1444	s = splbio()splraise(0x3);
1445	DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1446
1447	/* Save previous command register state */
1448	ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1449
1450	/* Clear ST, ignoring failure */
1451	ahci_port_stop(ap, 0);
1452
1453	/* Perform device detection */
1454	ahci_pwrite(ap, AHCI_PREG_SCTL0x2c, 0);
1455	delay(10000)(*delay_func)(10000);
1456	ahci_port_comreset(ap);
1457	splx(s)spllower(s);
1458	}
1459
1460	int
1461	ahci_port_portreset_poll(struct ahci_port *ap)
1462	{
1463	if ((ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) !=
1464	AHCI_PREG_SSTS_DET_DEV0x3)
1465	return (EAGAIN35);
1466	return (0);
1467	}
1468
1469	void
1470	ahci_port_portreset_wait(struct ahci_port *ap)
1471	{
1472	int i;
1473
1474	for (i = 0; i < 1000; i++) {
1475	if (ahci_port_portreset_poll(ap) == 0)
1476	break;
1477	delay(1000)(*delay_func)(1000);
1478	}
1479	}
1480
1481	int
1482	ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
1483	{
1484	int rc, s, retries = 0;
1485
1486	s = splbio()splraise(0x3);
1487	retry:
1488	if (ahci_port_portreset_poll(ap)) {
1489	rc = ENODEV19;
1490	if (ahci_pread(ap, AHCI_PREG_SSTS0x28) & AHCI_PREG_SSTS_DET0xf) {
1491	/* this may be a port multiplier with no device
1492	* on port 0, so still do the pmp check if requested.
1493	*/
1494	} else {
1495	goto err;
1496	}
1497	} else {
1498	/* Clear SERR (incl X bit), so TFD can update */
1499	ahci_pwrite(ap, AHCI_PREG_SERR0x30, ahci_pread(ap, AHCI_PREG_SERR0x30));
1500
1501	/* Wait for device to become ready */
1502	if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY \|ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (3))
1503	AHCI_PREG_TFD_STS_DRQ \| AHCI_PREG_TFD_STS_ERR, 3)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3) \| (1 <<0)), 0, (3))) {
1504	/* even if the device doesn't wake up, check if there's
1505	* a port multiplier there
1506	*/
1507	if (retries == 0) {
1508	retries = 1;
1509	ahci_port_comreset(ap);
1510	ahci_port_portreset_wait(ap);
1511	goto retry;
1512	}
1513	rc = EBUSY16;
1514	} else {
1515	rc = 0;
1516	}
1517	}
1518
1519	if (pmp != 0) {
1520	if (ahci_port_detect_pmp(ap)) {
1521	/* reset again without pmp support */
1522	pmp = 0;
1523	retries = 0;
1524	ahci_port_comreset(ap);
1525	ahci_port_portreset_wait(ap);
1526	goto retry;
1527	}
1528	}
1529
1530	err:
1531	/* Restore preserved port state */
1532	ahci_pwrite(ap, AHCI_PREG_CMD0x18, ap->ap_saved_cmd);
1533	ap->ap_saved_cmd = 0;
1534	splx(s)spllower(s);
1535
1536	return (rc);
1537	}
1538
1539	int
1540	ahci_port_portreset(struct ahci_port *ap, int pmp)
1541	{
1542	ahci_port_portreset_start(ap);
1543	ahci_port_portreset_wait(ap);
1544	return (ahci_port_portreset_finish(ap, pmp));
1545	}
1546
1547	int
1548	ahci_port_detect_pmp(struct ahci_port *ap)
1549	{
1550	int count, pmp_rc, rc;
1551	u_int32_t r, cmd;
1552	struct ahci_cmd_hdr *cmd_slot;
1553	struct ahci_ccb ccb = NULL((void )0);
1554	u_int8_t fis = NULL((void )0);
1555
1556	if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP(1<<2)) \|\|
1557	!ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)((ahci_read(ap->ap_sc, 0x000)) & ((1<<17)))) {
1558	return 0;
1559	}
1560
1561	rc = 0;
1562	pmp_rc = 0;
1563	count = 2;
1564	do {
1565	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
1566	count);
1567	if (ccb != NULL((void *)0)) {
1568	ahci_put_pmp_ccb(ccb);
1569	ccb = NULL((void *)0);
1570	}
1571	ahci_port_stop(ap, 0);
1572	ap->ap_state = AP_S_PMP_PROBE1;
1573
1574	/* set PMA in cmd reg */
1575	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1576	if ((cmd & AHCI_PREG_CMD_PMA(1<<17)) == 0) {
1577	cmd \|= AHCI_PREG_CMD_PMA(1<<17);
1578	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1579	}
1580
1581	/* Flush errors and request CLO unconditionally,
1582	* then start the port
1583	*/
1584	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1585	if (r & AHCI_PREG_SERR_DIAG_X(1<<10))
1586	ahci_pwrite(ap, AHCI_PREG_SERR0x30,
1587	AHCI_PREG_SERR_DIAG_X(1<<10));
1588
1589	/* Request CLO */
1590	ahci_port_clo(ap);
1591
1592	/* Clear port errors to permit TFD transfer */
1593	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1594	ahci_pwrite(ap, AHCI_PREG_SERR0x30, r);
1595
1596	/* Restart port */
1597	if (ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)))) {
1598	rc = EBUSY16;
1599	printf("%s: failed to start port, cannot probe PMP\n",
1600	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1601	break;
1602	}
1603
1604	/* Check whether CLO worked */
1605	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))
1606	AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))) {
1607	u_int32_t cap;
1608
1609	cap = ahci_read(ap->ap_sc, AHCI_REG_CAP0x000);
1610	printf("%s: CLO %s, need port reset\n",
1611	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
1612	ISSET(cap, AHCI_REG_CAP_SCLO)((cap) & ((1<<24)))
1613	? "failed" : "unsupported");
1614	pmp_rc = EBUSY16;
1615	break;
1616	}
1617
1618	/* Prep first command with SRST feature &
1619	* clear busy/reset flags
1620	*/
1621	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
1622	cmd_slot = ccb->ccb_cmd_hdr;
1623	memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)))
1624	sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1625
1626	fis = ccb->ccb_cmd_table->cfis;
1627	fis[0] = ATA_FIS_TYPE_H2D0x27;
1628	fis[1] = SATA_PMP_CONTROL_PORT0x0f;
1629	fis[15] = ATA_FIS_CONTROL_SRST0x04 \| ATA_FIS_CONTROL_4BIT0x08;
1630
1631	cmd_slot->prdtl = 0;
1632	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| 0xf000)))
1633	AHCI_CMD_LIST_FLAG_C \| AHCI_CMD_LIST_FLAG_R \|((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| 0xf000)))
1634	AHCI_CMD_LIST_FLAG_PMP)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (1<<10) \| (1<<8) \| 0xf000)));
1635
1636	DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
1637	PORTNAME(ap));
1638	ccb->ccb_xa.state = ATA_S_PENDING1;
1639	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1640	DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
1641	PORTNAME(ap));
1642	pmp_rc = EBUSY16;
1643	continue;
1644	}
1645
1646	if (ahci_pwait_clr(ap, AHCI_PREG_TFD,ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))
1647	AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ, 1)ahci_pwait_eq((ap), (0x20), ((1<<7) \| (1<<3)), 0, (1))) {
1648	printf("%s: port busy after first PMP probe FIS\n",
1649	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
1650	}
1651
1652	/* clear errors in case the device
1653	* didn't reset cleanly
1654	*/
1655	ahci_flush_tfd(ap);
1656	r = ahci_pread(ap, AHCI_PREG_SERR0x30);
1657	ahci_pwrite(ap, AHCI_PREG_SERR0x30, r);
1658
1659	/* Prep second command to read status and
1660	* complete reset sequence
1661	*/
1662	memset(ccb->ccb_cmd_table, 0,__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)))
1663	sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
1664	fis[0] = ATA_FIS_TYPE_H2D0x27;
1665	fis[1] = SATA_PMP_CONTROL_PORT0x0f;
1666	fis[15] = ATA_FIS_CONTROL_4BIT0x08;
1667
1668	cmd_slot->prdtl = 0;
1669	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| 0xf000)))
1670	AHCI_CMD_LIST_FLAG_PMP)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| 0xf000)));
1671
1672	DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
1673	PORTNAME(ap));
1674	ccb->ccb_xa.state = ATA_S_PENDING1;
1675	if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1676	DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
1677	"cmd failed\n", PORTNAME(ap));
1678	pmp_rc = EBUSY16;
1679	continue;
1680	}
1681
1682	/* apparently we need to retry at least once
1683	* to get the right signature
1684	*/
1685	fis[15] = 0;
1686	pmp_rc = 0;
1687	} while (--count);
1688
1689	if (ccb != NULL((void *)0)) {
1690	ahci_put_pmp_ccb(ccb);
1691	ccb = NULL((void *)0);
1692	}
1693
1694	if (ap->ap_state == AP_S_PMP_PROBE1) {
1695	ap->ap_state = AP_S_NORMAL0;
1696	}
1697
1698	if (pmp_rc == 0) {
1699	if (ahci_port_signature(ap) != ATA_PORT_T_PM3) {
1700	DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
1701	PORTNAME(ap));
1702	pmp_rc = EBUSY16;
1703	} else {
1704	DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
1705	PORTNAME(ap));
1706	}
1707	}
1708
1709	if (pmp_rc == 0) {
1710	if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
1711	pmp_rc = EBUSY16;
1712	} else {
1713	rc = 0;
1714	}
1715	}
1716
1717	/* if PMP detection failed, so turn off the PMA bit and
1718	* reset the port again
1719	*/
1720	if (pmp_rc != 0) {
1721	DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
1722	"the port\n", PORTNAME(ap));
1723	ahci_port_stop(ap, 0);
1724	ahci_port_clo(ap);
1725	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
1726	cmd &= ~AHCI_PREG_CMD_PMA(1<<17);
1727	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
1728
1729	ahci_pwrite(ap, AHCI_PREG_IE0x14, 0);
1730	ahci_port_stop(ap, 0);
1731	if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF(1<<29))
1732	ahci_pwrite(ap, AHCI_PREG_SNTF0x3c, -1);
1733	ahci_flush_tfd(ap);
1734	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
1735
1736	ahci_pwrite(ap, AHCI_PREG_IS0x10, -1);
1737
1738	ahci_enable_interrupts(ap);
1739
1740	rc = pmp_rc;
1741	}
1742
1743	return (rc);
1744	}
1745
1746	void
1747	ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
1748	u_int32_t flags)
1749	{
1750	flags \|= len - 1;
1751
1752	htolem64(&prd->dba, addr)((__uint64_t )(&prd->dba) = ((__uint64_t)(addr)));
1753	htolem32(&prd->flags, flags)((__uint32_t )(&prd->flags) = ((__uint32_t)(flags)));
1754	}
1755
1756	int
1757	ahci_load_prdt(struct ahci_ccb *ccb)
1758	{
1759	struct ahci_port *ap = ccb->ccb_port;
1760	struct ahci_softc *sc = ap->ap_sc;
1761	struct ata_xfer *xa = &ccb->ccb_xa;
1762	struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
1763	bus_dmamap_t dmap = ccb->ccb_dmamap;
1764	struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1765	int i, error;
1766
1767	if (xa->datalen == 0) {
1768	ccb->ccb_cmd_hdr->prdtl = 0;
1769	return (0);
1770	}
1771
1772	error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap) , (xa->data), (xa->datalen), (((void )0)), ((xa->flags & (1<<2)) ? 0x0001 : 0x0000))
1773	(xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK)((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (dmap) , (xa->data), (xa->datalen), (((void )0)), ((xa->flags & (1<<2)) ? 0x0001 : 0x0000));
1774	if (error != 0) {
1775	printf("%s: error %d loading dmamap\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), error);
1776	return (1);
1777	}
1778
1779	for (i = 0; i < dmap->dm_nsegs - 1; i++) {
1780	ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
1781	dmap->dm_segs[i].ds_len, 0);
1782	}
1783
1784	ahci_load_prdt_seg(&prdt[i],
1785	dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
1786	ISSET(xa->flags, ATA_F_PIO)((xa->flags) & ((1<<4))) ? AHCI_PRDT_FLAG_INTR(1<<31) : 0);
1787
1788	htolem16(&cmd_slot->prdtl, dmap->dm_nsegs)((__uint16_t )(&cmd_slot->prdtl) = ((__uint16_t)(dmap ->dm_nsegs)));
1789
1790	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04))
1791	(xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04))
1792	BUS_DMASYNC_PREWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x01 : 0x04));
1793
1794	return (0);
1795	}
1796
1797	void
1798	ahci_unload_prdt(struct ahci_ccb *ccb)
1799	{
1800	struct ahci_port *ap = ccb->ccb_port;
1801	struct ahci_softc *sc = ap->ap_sc;
1802	struct ata_xfer *xa = &ccb->ccb_xa;
1803	bus_dmamap_t dmap = ccb->ccb_dmamap;
1804
1805	if (xa->datalen != 0) {
1806	bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08))
1807	(xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08))
1808	BUS_DMASYNC_POSTWRITE)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (dmap) , (0), (dmap->dm_mapsize), ((xa->flags & (1<< 0)) ? 0x02 : 0x08));
1809
1810	bus_dmamap_unload(sc->sc_dmat, dmap)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (dmap ));
1811
1812	if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6))
1813	xa->resid = 0;
1814	else
1815	xa->resid = xa->datalen -
1816	lemtoh32(&ccb->ccb_cmd_hdr->prdbc)((__uint32_t)((__uint32_t )(&ccb->ccb_cmd_hdr->prdbc )));
1817	}
1818	}
1819
1820	int
1821	ahci_poll(struct ahci_ccb ccb, int timeout, void (timeout_fn)(void *))
1822	{
1823	struct ahci_port *ap = ccb->ccb_port;
1824	int s;
1825
1826	s = splbio()splraise(0x3);
1827	ahci_start(ccb);
1828	do {
1829	if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot ))
1830	1 << ccb->ccb_slot)((ahci_port_intr(ap, 0xffffffff)) & (1 << ccb->ccb_slot ))) {
1831	splx(s)spllower(s);
1832	return (0);
1833	}
1834	if (ccb->ccb_xa.state == ATA_S_ERROR3) {
1835	DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
1836	PORTNAME(ap), ccb->ccb_slot);
1837	/* pretend it timed out? */
1838	if (timeout_fn != NULL((void *)0)) {
1839	timeout_fn(ccb);
1840	}
1841	splx(s)spllower(s);
1842	return (1);
1843	}
1844
1845	delay(1000)(*delay_func)(1000);
1846	} while (--timeout > 0);
1847
1848	/* Run timeout while at splbio, otherwise ahci_intr could interfere. */
1849	if (timeout_fn != NULL((void *)0))
1850	timeout_fn(ccb);
1851
1852	splx(s)spllower(s);
1853
1854	return (1);
1855	}
1856
1857	void
1858	ahci_start(struct ahci_ccb *ccb)
1859	{
1860	struct ahci_port *ap = ccb->ccb_port;
1861	struct ahci_softc *sc = ap->ap_sc;
1862
1863	/* Zero transferred byte count before transfer */
1864	ccb->ccb_cmd_hdr->prdbc = 0;
1865
1866	/* Sync command list entry and corresponding command table entry */
1867	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) )
1868	ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) )
1869	sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x04) );
1870	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 ))
1871	ccb->ccb_slot * sizeof(struct ahci_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 ))
1872	sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x04 ));
1873
1874	/* Prepare RFIS area for write by controller */
1875	bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x01))
1876	sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x01));
1877
1878	if (ccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) {
1879	/* Issue NCQ commands only when there are no outstanding
1880	* standard commands. */
1881	if (ap->ap_active != 0 \|\| !TAILQ_EMPTY(&ap->ap_ccb_pending)(((&ap->ap_ccb_pending)->tqh_first) == ((void *)0)) \|\|
1882	(ap->ap_sactive != 0 &&
1883	ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
1884	TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_pending)->tqh_last; (& ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
1885	} else {
1886	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1886, "ap->ap_active_cnt == 0" ));
1887	ap->ap_sactive \|= (1 << ccb->ccb_slot);
1888	ccb->ccb_xa.state = ATA_S_ONCHIP5;
1889	ahci_pwrite(ap, AHCI_PREG_SACT0x34, 1 << ccb->ccb_slot);
1890	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
1891	ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
1892	}
1893	} else {
1894	/* Wait for all NCQ commands to finish before issuing standard
1895	* command. */
1896	if (ap->ap_sactive != 0 \|\| ap->ap_active_cnt == 2)
1897	TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_pending)->tqh_last; (& ap->ap_ccb_pending)->tqh_last = (ccb); (&ap->ap_ccb_pending )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
1898	else if (ap->ap_active_cnt < 2) {
1899	ap->ap_active \|= 1 << ccb->ccb_slot;
1900	ccb->ccb_xa.state = ATA_S_ONCHIP5;
1901	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
1902	ap->ap_active_cnt++;
1903	}
1904	}
1905	}
1906
1907	void
1908	ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1909	{
1910	struct ahci_ccb *nextccb;
1911	u_int32_t sact_change = 0;
1912
1913	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1913, "ap->ap_active_cnt == 0" ));
1914
1915	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1916	if (nextccb == NULL((void *)0) \|\| !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)))
1917	return;
1918
1919	/* Start all the NCQ commands at the head of the pending list.
1920	* If a port multiplier is attached to the port, we can only
1921	* issue commands for one of its ports at a time.
1922	*/
1923	if (ap->ap_sactive != 0 &&
1924	ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
1925	return;
1926	}
1927
1928	ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
1929	do {
1930	TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void )0)) ( nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb )->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)-> tqh_last = (nextccb)->ccb_entry.tqe_prev; (nextccb)->ccb_entry .tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)-> ccb_entry.tqe_prev) = ((void )-1); ((nextccb)->ccb_entry. tqe_next) = ((void )-1); } while (0);
1931	sact_change \|= 1 << nextccb->ccb_slot;
1932	nextccb->ccb_xa.state = ATA_S_ONCHIP5;
1933	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1934	} while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)) &&
1935	(nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
1936
1937	ap->ap_sactive \|= sact_change;
1938	ahci_pwrite(ap, AHCI_PREG_SACT0x34, sact_change);
1939	ahci_pwrite(ap, AHCI_PREG_CI0x38, sact_change);
1940	}
1941
1942	void
1943	ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
1944	{
1945	struct ahci_ccb *nextccb;
1946
1947	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1948	if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6))) {
1949	if (last_was_ncq) {
1950	KASSERT(nextccb->ccb_xa.pmp_port !=((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void )0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951 , "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port"))
1951	ap->ap_pmp_ncq_port)((nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port) ? (void )0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c", 1951 , "nextccb->ccb_xa.pmp_port != ap->ap_pmp_ncq_port"));
1952	/* otherwise it should have been started already */
1953	} else {
1954	ap->ap_active_cnt--;
1955	}
1956
1957	/* Issue NCQ commands only when there are no outstanding
1958	* standard commands, and previous NCQ commands for other
1959	* PMP ports have finished.
1960	*/
1961	if (ap->ap_active == 0)
1962	ahci_issue_pending_ncq_commands(ap);
1963	else
1964	KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1964, "ap->ap_active_cnt == 1" ));
1965	} else if (nextccb) {
1966	if (ap->ap_sactive != 0 \|\| last_was_ncq)
1967	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1967, "ap->ap_active_cnt == 0" ));
1968
1969	/* Wait for all NCQ commands to finish before issuing standard
1970	* command. */
1971	if (ap->ap_sactive != 0)
1972	return;
1973
1974	/* Keep up to 2 standard commands on-chip at a time. */
1975	do {
1976	TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry)do { if (((nextccb)->ccb_entry.tqe_next) != ((void )0)) ( nextccb)->ccb_entry.tqe_next->ccb_entry.tqe_prev = (nextccb )->ccb_entry.tqe_prev; else (&ap->ap_ccb_pending)-> tqh_last = (nextccb)->ccb_entry.tqe_prev; (nextccb)->ccb_entry .tqe_prev = (nextccb)->ccb_entry.tqe_next; ((nextccb)-> ccb_entry.tqe_prev) = ((void )-1); ((nextccb)->ccb_entry. tqe_next) = ((void )-1); } while (0);
1977	ap->ap_active \|= 1 << nextccb->ccb_slot;
1978	nextccb->ccb_xa.state = ATA_S_ONCHIP5;
1979	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << nextccb->ccb_slot);
1980	if (last_was_ncq)
1981	ap->ap_active_cnt++;
1982	if (ap->ap_active_cnt == 2)
1983	break;
1984	KASSERT(ap->ap_active_cnt == 1)((ap->ap_active_cnt == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1984, "ap->ap_active_cnt == 1" ));
1985	nextccb = TAILQ_FIRST(&ap->ap_ccb_pending)((&ap->ap_ccb_pending)->tqh_first);
1986	} while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ(1<<6)));
1987	} else if (!last_was_ncq) {
1988	KASSERT(ap->ap_active_cnt == 1 \|\| ap->ap_active_cnt == 2)((ap->ap_active_cnt == 1 \|\| ap->ap_active_cnt == 2) ? ( void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 1988, "ap->ap_active_cnt == 1 \|\| ap->ap_active_cnt == 2" ));
1989
1990	/* Standard command finished, none waiting to start. */
1991	ap->ap_active_cnt--;
1992	} else {
1993	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 1993, "ap->ap_active_cnt == 0" ));
1994
1995	/* NCQ command finished. */
1996	}
1997	}
1998
1999	int
2000	ahci_intr(void *arg)
2001	{
2002	struct ahci_softc *sc = arg;
2003	u_int32_t is, ack = 0;
2004	int port;
2005
2006	/* Read global interrupt status */
2007	is = ahci_read(sc, AHCI_REG_IS0x008);
2008	if (is == 0 \|\| is == 0xffffffff)
2009	return (0);
2010	ack = is;
2011
2012	#ifdef AHCI_COALESCE
2013	/* Check coalescing interrupt first */
2014	if (is & sc->sc_ccc_mask) {
2015	DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2016	DEVNAME(sc));
2017	is &= ~sc->sc_ccc_mask;
2018	is \|= sc->sc_ccc_ports_cur;
2019	}
2020	#endif
2021
2022	/* Process interrupts for each port */
2023	while (is) {
2024	port = ffs(is) - 1;
2025	if (sc->sc_ports[port])
2026	ahci_port_intr(sc->sc_ports[port],
2027	AHCI_PREG_CI_ALL_SLOTS0xffffffff);
2028	is &= ~(1 << port);
2029	}
2030
2031	/* Finally, acknowledge global interrupt */
2032	ahci_write(sc, AHCI_REG_IS0x008, ack);
2033
2034	return (1);
2035	}
2036
2037	u_int32_t
2038	ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
2039	{
2040	struct ahci_softc *sc = ap->ap_sc;
2041	u_int32_t is, ci_saved, ci_masked, processed = 0;
2042	int slot, need_restart = 0;
2043	int process_error = 0;
2044	struct ahci_ccb *ccb;
2045	volatile u_int32_t *active;
2046	#ifdef DIAGNOSTIC1
2047	u_int32_t tmp;
2048	#endif
2049
2050	is = ahci_pread(ap, AHCI_PREG_IS0x10);
2051
2052	/* Ack port interrupt only if checking all command slots. */
2053	if (ci_mask == AHCI_PREG_CI_ALL_SLOTS0xffffffff)
2054	ahci_pwrite(ap, AHCI_PREG_IS0x10, is);
2055
2056	if (is)
2057	DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
2058	is, AHCI_PFMT_IS);
2059
2060	if (ap->ap_sactive) {
2061	/* Active NCQ commands - use SActive instead of CI */
2062	KASSERT(ap->ap_active == 0)((ap->ap_active == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2062, "ap->ap_active == 0"));
2063	KASSERT(ap->ap_active_cnt == 0)((ap->ap_active_cnt == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2063, "ap->ap_active_cnt == 0" ));
2064	ci_saved = ahci_pread(ap, AHCI_PREG_SACT0x34);
2065	active = &ap->ap_sactive;
2066	} else {
2067	/* Save CI */
2068	ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38);
2069	active = &ap->ap_active;
2070	}
2071
2072	if (is & AHCI_PREG_IS_TFES(1<<30)) {
2073	process_error = 1;
2074	} else if (is & AHCI_PREG_IS_DHRS(1<<0)) {
2075	u_int32_t tfd;
2076	u_int32_t cmd;
2077	u_int32_t serr;
2078
2079	tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
2080	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
2081	serr = ahci_pread(ap, AHCI_PREG_SERR0x30);
2082	if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) &&
2083	(cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) {
2084	DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
2085	" %b, DIAG: %b\n", PORTNAME(ap), tfd,
2086	AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2087	AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2088	AHCI_PFMT_SERR_DIAG);
2089	process_error = 1;
2090	} else {
2091	/* rfis copy back is in the normal execution path */
2092	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0));
2093	}
2094	}
2095
2096	/* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
2097	if (process_error) {
2098	u_int32_t tfd, serr;
2099	int err_slot;
2100
2101	tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
2102	serr = ahci_pread(ap, AHCI_PREG_SERR0x30);
2103
2104	if (ap->ap_sactive == 0) {
2105	/* Errored slot is easy to determine from CMD. */
2106	err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,(((ahci_pread(ap, 0x18)) >> 8) & 0x1f)
2107	AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f);
2108
2109	if ((ci_saved & (1 << err_slot)) == 0) {
2110	/*
2111	* Hardware doesn't seem to report correct
2112	* slot number. If there's only one
2113	* outstanding command we can cope,
2114	* otherwise fail all active commands.
2115	*/
2116	if (ap->ap_active_cnt == 1)
2117	err_slot = ffs(ap->ap_active) - 1;
2118	else
2119	goto failall;
2120	}
2121
2122	ccb = &ap->ap_ccbs[err_slot];
2123
2124	/* Preserve received taskfile data from the RFIS. */
2125	memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)))
2126	sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)));
2127	} else
2128	err_slot = -1; /* Must extract error from log page */
2129
2130	DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
2131	" %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
2132	AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2133	AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2134	AHCI_PFMT_SERR_DIAG);
2135
2136	/* Turn off ST to clear CI and SACT. */
2137	ahci_port_stop(ap, 0);
2138	need_restart = 1;
2139
2140	/* Clear SERR to enable capturing new errors. */
2141	ahci_pwrite(ap, AHCI_PREG_SERR0x30, serr);
2142
2143	/* Acknowledge the interrupts we can recover from. */
2144	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_TFES(1<<30) \|
2145	AHCI_PREG_IS_IFS(1<<27));
2146	is = ahci_pread(ap, AHCI_PREG_IS0x10);
2147
2148	/* If device hasn't cleared its busy status, try to idle it. */
2149	if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY \| AHCI_PREG_TFD_STS_DRQ)((tfd) & ((1<<7) \| (1<<3)))) {
2150
2151	if ((ap->ap_state == AP_S_PMP_PORT_PROBE2) \|\|
2152	(ap->ap_state == AP_S_ERROR_RECOVERY3)) {
2153	/* can't reset the port here, just make sure
2154	* the operation fails and the port still works.
2155	*/
2156	} else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
2157	printf("%s: error on PMP port %d, idling "
2158	"device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2159	ccb->ccb_xa.pmp_port);
2160	if (ahci_pmp_port_softreset(ap,
2161	ccb->ccb_xa.pmp_port) == 0) {
2162	printf("%s: unable to softreset port "
2163	"%d\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2164	ccb->ccb_xa.pmp_port);
2165	if (ahci_pmp_port_portreset(ap,
2166	ccb->ccb_xa.pmp_port)) {
2167	printf("%s: failed to port "
2168	" reset %d, giving up on "
2169	"it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname),
2170	ccb->ccb_xa.pmp_port);
2171	goto fatal;
2172	}
2173	}
2174	} else {
2175	printf("%s: attempting to idle device\n",
2176	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2177	if (ahci_port_softreset(ap)) {
2178	printf("%s: failed to soft reset "
2179	"device\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2180	if (ahci_port_portreset(ap, 0)) {
2181	printf("%s: failed to port "
2182	"reset device, give up on "
2183	"it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2184	goto fatal;
2185	}
2186	}
2187	}
2188
2189	/* Had to reset device, can't gather extended info. */
2190	} else if (ap->ap_sactive) {
2191	/* Recover the NCQ error from log page 10h.
2192	* We can only have queued commands active for one port
2193	* at a time, so we know which device errored.
2194	*/
2195	ahci_port_read_ncq_error(ap, &err_slot,
2196	ap->ap_pmp_ncq_port);
2197	if (err_slot < 0)
2198	goto failall;
2199
2200	DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2201	PORTNAME(ap), err_slot);
2202
2203	ccb = &ap->ap_ccbs[err_slot];
2204	if (ccb->ccb_xa.state != ATA_S_ONCHIP5) {
2205	printf("%s: NCQ errored slot %d is idle"
2206	" (%08x active)\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), err_slot,
2207	ci_saved);
2208	goto failall;
2209	}
2210	} else {
2211	/* Didn't reset, could gather extended info from log. */
2212	}
2213
2214	/*
2215	* If we couldn't determine the errored slot, reset the port
2216	* and fail all the active slots.
2217	*/
2218	if (err_slot == -1) {
2219	if (ahci_port_softreset(ap) != 0 &&
2220	ahci_port_portreset(ap, 0) != 0) {
2221	printf("%s: couldn't reset after NCQ error, "
2222	"disabling device.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2223	goto fatal;
2224	}
2225	printf("%s: couldn't recover NCQ error, failing "
2226	"all outstanding commands.\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2227	goto failall;
2228	}
2229
2230	/* Clear the failed command in saved CI so completion runs. */
2231	ci_saved &= ~(1 << err_slot);
2232
2233	/* Note the error in the ata_xfer. */
2234	KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2234, "ccb->ccb_xa.state == ATA_S_ONCHIP" ));
2235	ccb->ccb_xa.state = ATA_S_ERROR3;
2236
2237	#ifdef DIAGNOSTIC1
2238	/* There may only be one outstanding standard command now. */
2239	if (ap->ap_sactive == 0) {
2240	tmp = ci_saved;
2241	if (tmp) {
2242	slot = ffs(tmp) - 1;
2243	tmp &= ~(1 << slot);
2244	KASSERT(tmp == 0)((tmp == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2244, "tmp == 0"));
2245	}
2246	}
2247	#endif
2248	}
2249
2250	/* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts
2251	* when there's a normal SATA device attached.
2252	*/
2253	if ((ap->ap_state == AP_S_PMP_PROBE1) &&
2254	(ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE(1<<1)) &&
2255	(is & AHCI_PREG_IS_IPMS(1<<23))) {
2256	slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD))(((ahci_pread(ap, 0x18)) >> 8) & 0x1f);
2257	DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
2258	PORTNAME(ap), slot);
2259
2260	ccb = &ap->ap_ccbs[slot];
2261	ccb->ccb_xa.state = ATA_S_ERROR3;
2262
2263	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IPMS(1<<23));
2264	is &= ~AHCI_PREG_IS_IPMS(1<<23);
2265	}
2266
2267	/* ignore IFS errors while resetting a PMP port */
2268	if ((is & AHCI_PREG_IS_IFS(1<<27)) /&& ap->ap_pmp_ignore_ifs/) {
2269	DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
2270	"port\n", PORTNAME(ap));
2271
2272	need_restart = 1;
2273	ahci_pwrite(ap, AHCI_PREG_SERR0x30, -1);
2274	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_IFS(1<<27));
2275	is &= ~AHCI_PREG_IS_IFS(1<<27);
	Value stored to 'is' is never read
2276	goto failall;
2277	}
2278
2279	/* Check for remaining errors - they are fatal. */
2280	if (is & (AHCI_PREG_IS_TFES(1<<30) \| AHCI_PREG_IS_HBFS(1<<29) \| AHCI_PREG_IS_IFS(1<<27) \|
2281	AHCI_PREG_IS_OFS(1<<24) \| AHCI_PREG_IS_UFS(1<<4))) {
2282	printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
2283	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), is, AHCI_PFMT_IS"\20" "\040CPDS" "\037TFES" "\036HBFS" "\035HBDS" "\034IFS" "\033INFS" "\031OFS" "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" "\007PCS" "\005UFS" "\004SDBS" "\003DSS" "\002PSS" "\001DHRS");
2284
2285	/* XXX try recovery first */
2286	goto fatal;
2287	}
2288
2289	/* Fail all outstanding commands if we know the port won't recover. */
2290	if (ap->ap_state == AP_S_FATAL_ERROR4) {
2291	fatal:
2292	ap->ap_state = AP_S_FATAL_ERROR4;
2293	failall:
2294
2295	/* Ensure port is shut down. */
2296	ahci_port_stop(ap, 1);
2297
2298	/* Error all the active slots. */
2299	ci_masked = ci_saved & *active;
2300	while (ci_masked) {
2301	slot = ffs(ci_masked) - 1;
2302	ccb = &ap->ap_ccbs[slot];
2303	ci_masked &= ~(1 << slot);
2304	ccb->ccb_xa.state = ATA_S_ERROR3;
2305	}
2306
2307	/* Run completion for all active slots. */
2308	ci_saved &= ~*active;
2309
2310	/* Don't restart the port if our problems were deemed fatal. */
2311	if (ap->ap_state == AP_S_FATAL_ERROR4)
2312	need_restart = 0;
2313	}
2314
2315	/*
2316	* CCB completion is detected by noticing its slot's bit in CI has
2317	* changed to zero some time after we activated it.
2318	* If we are polling, we may only be interested in particular slot(s).
2319	*/
2320	ci_masked = ~ci_saved & *active & ci_mask;
2321	while (ci_masked) {
2322	slot = ffs(ci_masked) - 1;
2323	ccb = &ap->ap_ccbs[slot];
2324	ci_masked &= ~(1 << slot);
2325
2326	DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2327	PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2328	" (error)" : "");
2329
2330	bus_dmamap_sync(sc->sc_dmat,((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) )
2331	AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) )
2332	ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) )
2333	sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_list)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_hdr)), (sizeof(struct ahci_cmd_hdr)), (0x08) );
2334
2335	bus_dmamap_sync(sc->sc_dmat,((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ))
2336	AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ))
2337	ccb->ccb_slot * sizeof(struct ahci_cmd_table),((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ))
2338	sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE)((sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_cmd_table)->adm_map)), (ccb->ccb_slot sizeof (struct ahci_cmd_table)), (sizeof(struct ahci_cmd_table)), (0x08 ));
2339
2340	bus_dmamap_sync(sc->sc_dmat,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02))
2341	AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02))
2342	sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD)(*(sc->sc_dmat)->_dmamap_sync)((sc->sc_dmat), (((ap-> ap_dmamem_rfis)->adm_map)), (0), (sizeof(struct ahci_rfis) ), (0x02));
2343
2344	*active &= ~(1 << ccb->ccb_slot);
2345	/* Copy the rfis into the ccb if we were asked for it */
2346	if (ccb->ccb_xa.state == ATA_S_ONCHIP5 &&
2347	ccb->ccb_xa.flags & ATA_F_GET_RFIS(1<<8)) {
2348	memcpy(&ccb->ccb_xa.rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)))
2349	ap->ap_rfis->rfis,__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)))
2350	sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (ap->ap_rfis-> rfis), (sizeof(struct ata_fis_d2h)));
2351	}
2352
2353	ccb->ccb_done(ccb);
2354
2355	processed \|= 1 << ccb->ccb_slot;
2356	}
2357
2358	if (need_restart) {
2359	/* Restart command DMA on the port */
2360	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
2361
2362	/* Re-enable outstanding commands on port. */
2363	if (ci_saved) {
2364	#ifdef DIAGNOSTIC1
2365	tmp = ci_saved;
2366	while (tmp) {
2367	slot = ffs(tmp) - 1;
2368	tmp &= ~(1 << slot);
2369	ccb = &ap->ap_ccbs[slot];
2370	KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP)((ccb->ccb_xa.state == 5) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2370, "ccb->ccb_xa.state == ATA_S_ONCHIP" ));
2371	KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap-> ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)" ))
2372	(!!ap->ap_sactive))(((!!(ccb->ccb_xa.flags & (1<<6))) == (!!ap-> ap_sactive)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2372, "(!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == (!!ap->ap_sactive)" ));
2373	}
2374	#endif
2375	DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2376	"re-enabling%s slots %08x\n", PORTNAME(ap),
2377	ap->ap_sactive ? " NCQ" : "", ci_saved);
2378
2379	if (ap->ap_sactive)
2380	ahci_pwrite(ap, AHCI_PREG_SACT0x34, ci_saved);
2381	ahci_pwrite(ap, AHCI_PREG_CI0x38, ci_saved);
2382	}
2383	}
2384
2385	return (processed);
2386	}
2387
2388	struct ahci_ccb *
2389	ahci_get_ccb(struct ahci_port *ap)
2390	{
2391	struct ahci_ccb *ccb;
2392
2393	mtx_enter(&ap->ap_ccb_mtx);
2394	ccb = TAILQ_FIRST(&ap->ap_ccb_free)((&ap->ap_ccb_free)->tqh_first);
2395	if (ccb != NULL((void *)0)) {
2396	KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2396, "ccb->ccb_xa.state == ATA_S_PUT" ));
2397	TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb )->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = (ccb )->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = (( void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
2398	ccb->ccb_xa.state = ATA_S_SETUP0;
2399	}
2400	mtx_leave(&ap->ap_ccb_mtx);
2401
2402	return (ccb);
2403	}
2404
2405	void
2406	ahci_put_ccb(struct ahci_ccb *ccb)
2407	{
2408	struct ahci_port *ap = ccb->ccb_port;
2409
2410	#ifdef DIAGNOSTIC1
2411	if (ccb->ccb_xa.state != ATA_S_COMPLETE2 &&
2412	ccb->ccb_xa.state != ATA_S_TIMEOUT4 &&
2413	ccb->ccb_xa.state != ATA_S_ERROR3) {
2414	printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2415	"slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), ccb->ccb_xa.state,
2416	ccb->ccb_slot);
2417	}
2418	#endif
2419
2420	ccb->ccb_xa.state = ATA_S_PUT6;
2421	mtx_enter(&ap->ap_ccb_mtx);
2422	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_free)->tqh_last; (&ap ->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
2423	mtx_leave(&ap->ap_ccb_mtx);
2424	}
2425
2426	struct ahci_ccb *
2427	ahci_get_err_ccb(struct ahci_port *ap)
2428	{
2429	struct ahci_ccb *err_ccb;
2430	u_int32_t sact;
2431
2432	splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__ ); } } while (0);
2433
2434	/* No commands may be active on the chip. */
2435	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2436	if (sact != 0)
2437	printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2438	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2438, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2439
2440	#ifdef DIAGNOSTIC1
2441	KASSERT(ap->ap_err_busy == 0)((ap->ap_err_busy == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2441, "ap->ap_err_busy == 0" ));
2442	ap->ap_err_busy = 1;
2443	#endif
2444	/* Save outstanding command state. */
2445	ap->ap_err_saved_active = ap->ap_active;
2446	ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2447	ap->ap_err_saved_sactive = ap->ap_sactive;
2448
2449	/*
2450	* Pretend we have no commands outstanding, so that completions won't
2451	* run prematurely.
2452	*/
2453	ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2454
2455	/*
2456	* Grab a CCB to use for error recovery. This should never fail, as
2457	* we ask atascsi to reserve one for us at init time.
2458	*/
2459	err_ccb = ap->ap_ccb_err;
2460	err_ccb->ccb_xa.flags = 0;
2461	err_ccb->ccb_xa.state = ATA_S_SETUP0;
2462	err_ccb->ccb_done = ahci_empty_done;
2463
2464	return (err_ccb);
2465	}
2466
2467	void
2468	ahci_put_err_ccb(struct ahci_ccb *ccb)
2469	{
2470	struct ahci_port *ap = ccb->ccb_port;
2471	u_int32_t sact;
2472
2473	splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__ ); } } while (0);
2474
2475	#ifdef DIAGNOSTIC1
2476	KASSERT(ap->ap_err_busy)((ap->ap_err_busy) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/ic/ahci.c" , 2476, "ap->ap_err_busy"));
2477	#endif
2478	/* No commands may be active on the chip */
2479	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2480	if (sact != 0)
2481	printf("ahci_put_err_ccb but SACT %08x != 0?\n", sact);
2482	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2482, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2483
2484	/* Done with the CCB */
2485	KASSERT(ccb == ap->ap_ccb_err)((ccb == ap->ap_ccb_err) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2485, "ccb == ap->ap_ccb_err" ));
2486
2487	/* Restore outstanding command state */
2488	ap->ap_sactive = ap->ap_err_saved_sactive;
2489	ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2490	ap->ap_active = ap->ap_err_saved_active;
2491
2492	#ifdef DIAGNOSTIC1
2493	ap->ap_err_busy = 0;
2494	#endif
2495	}
2496
2497	struct ahci_ccb *
2498	ahci_get_pmp_ccb(struct ahci_port *ap)
2499	{
2500	struct ahci_ccb *ccb;
2501	u_int32_t sact;
2502
2503	/* some PMP commands need to be issued on slot 1,
2504	* particularly the command that clears SRST and
2505	* fetches the device signature.
2506	*
2507	* ensure the chip is idle and ccb 1 is available.
2508	*/
2509	splassert(IPL_BIO)do { if (splassert_ctl > 0) { splassert_check(0x3, __func__ ); } } while (0);
2510
2511	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2512	if (sact != 0)
2513	printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
2514	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2514, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2515
2516	ccb = &ap->ap_ccbs[1];
2517	KASSERT(ccb->ccb_xa.state == ATA_S_PUT)((ccb->ccb_xa.state == 6) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2517, "ccb->ccb_xa.state == ATA_S_PUT" ));
2518	ccb->ccb_xa.flags = 0;
2519	ccb->ccb_done = ahci_pmp_cmd_done;
2520
2521	mtx_enter(&ap->ap_ccb_mtx);
2522	TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_free)->tqh_last = (ccb )->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = (ccb )->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = (( void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
2523	mtx_leave(&ap->ap_ccb_mtx);
2524
2525	return ccb;
2526	}
2527
2528	void
2529	ahci_put_pmp_ccb(struct ahci_ccb *ccb)
2530	{
2531	struct ahci_port *ap = ccb->ccb_port;
2532	u_int32_t sact;
2533
2534	/* make sure this is the right ccb */
2535	KASSERT(ccb == &ap->ap_ccbs[1])((ccb == &ap->ap_ccbs[1]) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2535, "ccb == &ap->ap_ccbs[1]" ));
2536
2537	/* No commands may be active on the chip */
2538	sact = ahci_pread(ap, AHCI_PREG_SACT0x34);
2539	if (sact != 0)
2540	printf("ahci_put_pmp_ccb but SACT %08x != 0?\n", sact);
2541	KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0)((ahci_pread(ap, 0x38) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2541, "ahci_pread(ap, AHCI_PREG_CI) == 0" ));
2542
2543	ccb->ccb_xa.state = ATA_S_PUT6;
2544	mtx_enter(&ap->ap_ccb_mtx);
2545	TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry)do { (ccb)->ccb_entry.tqe_next = ((void )0); (ccb)->ccb_entry .tqe_prev = (&ap->ap_ccb_free)->tqh_last; (&ap ->ap_ccb_free)->tqh_last = (ccb); (&ap->ap_ccb_free )->tqh_last = &(ccb)->ccb_entry.tqe_next; } while ( 0);
2546	mtx_leave(&ap->ap_ccb_mtx);
2547	}
2548
2549	int
2550	ahci_port_read_ncq_error(struct ahci_port ap, int err_slotp, int pmp_port)
2551	{
2552	struct ahci_ccb *ccb;
2553	struct ahci_cmd_hdr *cmd_slot;
2554	u_int32_t cmd;
2555	struct ata_fis_h2d *fis;
2556	int rc = EIO5, oldstate;
2557
2558	DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2559	oldstate = ap->ap_state;
2560	ap->ap_state = AP_S_ERROR_RECOVERY3;
2561
2562	/* Save command register state. */
2563	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
2564
2565	/* Port should have been idled already. Start it. */
2566	KASSERT((cmd & AHCI_PREG_CMD_CR) == 0)(((cmd & (1<<15)) == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/ic/ahci.c", 2566, "(cmd & AHCI_PREG_CMD_CR) == 0" ));
2567	ahci_port_start(ap, 0)((ap)->ap_sc->sc_port_start((ap), (0)));
2568
2569	/* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2570	ccb = ahci_get_err_ccb(ap);
2571	ccb->ccb_xa.flags = ATA_F_NOWAIT(1<<2) \| ATA_F_READ(1<<0) \| ATA_F_POLL(1<<3);
2572	ccb->ccb_xa.data = ap->ap_err_scratch;
2573	ccb->ccb_xa.datalen = 512;
2574	cmd_slot = ccb->ccb_cmd_hdr;
2575	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
2576
2577	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2578	fis->type = ATA_FIS_TYPE_H2D0x27;
2579	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| pmp_port;
2580	fis->command = ATA_C_READ_LOG_EXT0x2f;
2581	fis->lba_low = 0x10; /* queued error log page (10h) */
2582	fis->sector_count = 1; /* number of sectors (1) */
2583	fis->sector_count_exp = 0;
2584	fis->lba_mid = 0; /* starting offset */
2585	fis->lba_mid_exp = 0;
2586	fis->device = 0;
2587
2588	htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS / \|((__uint16_t *)(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))))
2589	(pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT))((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(5 \| (pmp_port << 12))));
2590
2591	if (ahci_load_prdt(ccb) != 0) {
2592	rc = ENOMEM12; /* XXX caller must abort all commands */
2593	goto err;
2594	}
2595
2596	ccb->ccb_xa.state = ATA_S_PENDING1;
2597	if (ahci_poll(ccb, 1000, NULL((void *)0)) != 0 \|\|
2598	ccb->ccb_xa.state == ATA_S_ERROR3)
2599	goto err;
2600
2601	rc = 0;
2602	err:
2603	/* Abort our command, if it failed, by stopping command DMA. */
2604	if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)((ap->ap_active) & (1 << ccb->ccb_slot))) {
2605	printf("%s: log page read failed, slot %d was still active.\n",
2606	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), ccb->ccb_slot);
2607	ahci_port_stop(ap, 0);
2608	}
2609
2610	/* Done with the error CCB now. */
2611	ahci_unload_prdt(ccb);
2612	ahci_put_err_ccb(ccb);
2613
2614	/* Extract failed register set and tags from the scratch space. */
2615	if (rc == 0) {
2616	struct ata_log_page_10h *log;
2617	int err_slot;
2618
2619	log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2620	if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)((log->err_regs.type) & (0x80))) {
2621	/* Not queued bit was set - wasn't an NCQ error? */
2622	printf("%s: read NCQ error page, but not an NCQ "
2623	"error?\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2624	rc = ESRCH3;
2625	} else {
2626	/* Copy back the log record as a D2H register FIS. */
2627	*err_slotp = err_slot = log->err_regs.type &
2628	ATA_LOG_10H_TYPE_TAG_MASK0x1f;
2629
2630	ccb = &ap->ap_ccbs[err_slot];
2631	memcpy(&ccb->ccb_xa.rfis, &log->err_regs,__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs ), (sizeof(struct ata_fis_d2h)))
2632	sizeof(struct ata_fis_d2h))__builtin_memcpy((&ccb->ccb_xa.rfis), (&log->err_regs ), (sizeof(struct ata_fis_d2h)));
2633	ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H0x34;
2634	ccb->ccb_xa.rfis.flags = 0;
2635	}
2636	}
2637
2638	/* Restore saved CMD register state */
2639	ahci_pwrite(ap, AHCI_PREG_CMD0x18, cmd);
2640	ap->ap_state = oldstate;
2641
2642	return (rc);
2643	}
2644
2645	struct ahci_dmamem *
2646	ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
2647	{
2648	struct ahci_dmamem *adm;
2649	int nsegs;
2650
2651	adm = malloc(sizeof(*adm), M_DEVBUF2, M_NOWAIT0x0002 \| M_ZERO0x0008);
2652	if (adm == NULL((void *)0))
2653	return (NULL((void *)0));
2654
2655	adm->adm_size = size;
2656
2657	if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size ), (1), (size), (0), (0x0001 \| 0x0002), (&adm->adm_map ))
2658	BUS_DMA_NOWAIT \| BUS_DMA_ALLOCNOW, &adm->adm_map)(*(sc->sc_dmat)->_dmamap_create)((sc->sc_dmat), (size ), (1), (size), (0), (0x0001 \| 0x0002), (&adm->adm_map )) != 0)
2659	goto admfree;
2660
2661	if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size ), ((1 << 12)), (0), (&adm->adm_seg), (1), (& nsegs), (0x0001 \| 0x1000))
2662	1, &nsegs, BUS_DMA_NOWAIT \| BUS_DMA_ZERO)(*(sc->sc_dmat)->_dmamem_alloc)((sc->sc_dmat), (size ), ((1 << 12)), (0), (&adm->adm_seg), (1), (& nsegs), (0x0001 \| 0x1000)) != 0)
2663	goto destroy;
2664
2665	if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm ->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001 \| 0x0004))
2666	&adm->adm_kva, BUS_DMA_NOWAIT \| BUS_DMA_COHERENT)(*(sc->sc_dmat)->_dmamem_map)((sc->sc_dmat), (&adm ->adm_seg), (nsegs), (size), (&adm->adm_kva), (0x0001 \| 0x0004)) != 0)
2667	goto free;
2668
2669	if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm-> adm_map), (adm->adm_kva), (size), (((void )0)), (0x0001))
2670	NULL, BUS_DMA_NOWAIT)((sc->sc_dmat)->_dmamap_load)((sc->sc_dmat), (adm-> adm_map), (adm->adm_kva), (size), (((void )0)), (0x0001)) != 0)
2671	goto unmap;
2672
2673	return (adm);
2674
2675	unmap:
2676	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm-> adm_kva), (size));
2677	free:
2678	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (& adm->adm_seg), (1));
2679	destroy:
2680	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm ->adm_map));
2681	admfree:
2682	free(adm, M_DEVBUF2, sizeof(*adm));
2683
2684	return (NULL((void *)0));
2685	}
2686
2687	void
2688	ahci_dmamem_free(struct ahci_softc sc, struct ahci_dmamem adm)
2689	{
2690	bus_dmamap_unload(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_unload)((sc->sc_dmat), (adm ->adm_map));
2691	bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size)(*(sc->sc_dmat)->_dmamem_unmap)((sc->sc_dmat), (adm-> adm_kva), (adm->adm_size));
2692	bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1)(*(sc->sc_dmat)->_dmamem_free)((sc->sc_dmat), (& adm->adm_seg), (1));
2693	bus_dmamap_destroy(sc->sc_dmat, adm->adm_map)(*(sc->sc_dmat)->_dmamap_destroy)((sc->sc_dmat), (adm ->adm_map));
2694	free(adm, M_DEVBUF2, sizeof(*adm));
2695	}
2696
2697	u_int32_t
2698	ahci_read(struct ahci_softc *sc, bus_size_t r)
2699	{
2700	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2701	BUS_SPACE_BARRIER_READ0x01);
2702	return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)((sc->sc_iot)->read_4((sc->sc_ioh), (r))));
2703	}
2704
2705	void
2706	ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2707	{
2708	bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v)((sc->sc_iot)->write_4((sc->sc_ioh), (r), (v)));
2709	bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2710	BUS_SPACE_BARRIER_WRITE0x02);
2711	}
2712
2713	int
2714	ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
2715	u_int32_t target)
2716	{
2717	int i;
2718
2719	for (i = 0; i < 1000; i++) {
2720	if ((ahci_read(sc, r) & mask) != target)
2721	return (0);
2722	delay(1000)(*delay_func)(1000);
2723	}
2724
2725	return (1);
2726	}
2727
2728	u_int32_t
2729	ahci_pread(struct ahci_port *ap, bus_size_t r)
2730	{
2731	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2732	BUS_SPACE_BARRIER_READ0x01);
2733	return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)((ap->ap_sc->sc_iot)->read_4((ap->ap_ioh), (r))));
2734	}
2735
2736	void
2737	ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2738	{
2739	bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v)((ap->ap_sc->sc_iot)->write_4((ap->ap_ioh), (r), ( v)));
2740	bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2741	BUS_SPACE_BARRIER_WRITE0x02);
2742	}
2743
2744	int
2745	ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
2746	u_int32_t target, int n)
2747	{
2748	int i;
2749
2750	for (i = 0; i < n * 1000; i++) {
2751	if ((ahci_pread(ap, r) & mask) == target)
2752	return (0);
2753	delay(1000)(*delay_func)(1000);
2754	}
2755
2756	return (1);
2757	}
2758
2759	int
2760	ahci_ata_probe(void *xsc, int port, int lun)
2761	{
2762	struct ahci_softc *sc = xsc;
2763	struct ahci_port *ap = sc->sc_ports[port];
2764
2765	if (ap == NULL((void *)0))
2766	return (ATA_PORT_T_NONE0);
2767
2768	if (lun != 0) {
2769	int pmp_port = lun - 1;
2770	if (pmp_port >= ap->ap_pmp_ports) {
2771	return (ATA_PORT_T_NONE0);
2772	}
2773	return (ahci_pmp_port_probe(ap, pmp_port));
2774	} else {
2775	return (ahci_port_signature(ap));
2776	}
2777	}
2778
2779	void
2780	ahci_ata_free(void *xsc, int port, int lun)
2781	{
2782
2783	}
2784
2785	struct ata_xfer *
2786	ahci_ata_get_xfer(void *aaa_cookie, int port)
2787	{
2788	struct ahci_softc *sc = aaa_cookie;
2789	struct ahci_port *ap = sc->sc_ports[port];
2790	struct ahci_ccb *ccb;
2791
2792	ccb = ahci_get_ccb(ap);
2793	if (ccb == NULL((void *)0)) {
2794	DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
2795	PORTNAME(ap));
2796	return (NULL((void *)0));
2797	}
2798
2799	DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
2800	PORTNAME(ap), ccb->ccb_slot);
2801
2802	return ((struct ata_xfer *)ccb);
2803	}
2804
2805	void
2806	ahci_ata_put_xfer(struct ata_xfer *xa)
2807	{
2808	struct ahci_ccb ccb = (struct ahci_ccb )xa;
2809
2810	DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
2811
2812	ahci_put_ccb(ccb);
2813	}
2814
2815	void
2816	ahci_ata_cmd(struct ata_xfer *xa)
2817	{
2818	struct ahci_ccb ccb = (struct ahci_ccb )xa;
2819	struct ahci_cmd_hdr *cmd_slot;
2820	int s;
2821	u_int16_t flags;
2822
2823	if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR4)
2824	goto failcmd;
2825
2826	ccb->ccb_done = ahci_ata_cmd_done;
2827
2828	cmd_slot = ccb->ccb_cmd_hdr;
2829	flags = 5 /* FIS length (in DWORDs) */;
2830	flags \|= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12;
2831
2832	if (xa->flags & ATA_F_WRITE(1<<1))
2833	flags \|= AHCI_CMD_LIST_FLAG_W(1<<6);
2834
2835	if (xa->flags & ATA_F_PACKET(1<<5))
2836	flags \|= AHCI_CMD_LIST_FLAG_A(1<<5);
2837
2838	htolem16(&cmd_slot->flags, flags)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(flags )));
2839
2840	if (ahci_load_prdt(ccb) != 0)
2841	goto failcmd;
2842
2843	timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
2844
2845	xa->state = ATA_S_PENDING1;
2846
2847	if (xa->flags & ATA_F_POLL(1<<3))
2848	ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
2849	else {
2850	s = splbio()splraise(0x3);
2851	timeout_add_msec(&xa->stimeout, xa->timeout);
2852	ahci_start(ccb);
2853	splx(s)spllower(s);
2854	}
2855
2856	return;
2857
2858	failcmd:
2859	s = splbio()splraise(0x3);
2860	xa->state = ATA_S_ERROR3;
2861	ata_complete(xa);
2862	splx(s)spllower(s);
2863	}
2864
2865	void
2866	ahci_pmp_cmd_done(struct ahci_ccb *ccb)
2867	{
2868	struct ata_xfer *xa = &ccb->ccb_xa;
2869
2870	if (xa->state == ATA_S_ONCHIP5 \|\| xa->state == ATA_S_ERROR3)
2871	ahci_issue_pending_commands(ccb->ccb_port,
2872	xa->flags & ATA_F_NCQ(1<<6));
2873
2874	xa->state = ATA_S_COMPLETE2;
2875	}
2876
2877
2878	void
2879	ahci_ata_cmd_done(struct ahci_ccb *ccb)
2880	{
2881	struct ata_xfer *xa = &ccb->ccb_xa;
2882
2883	timeout_del(&xa->stimeout);
2884
2885	if (xa->state == ATA_S_ONCHIP5 \|\| xa->state == ATA_S_ERROR3)
2886	ahci_issue_pending_commands(ccb->ccb_port,
2887	xa->flags & ATA_F_NCQ(1<<6));
2888
2889	ahci_unload_prdt(ccb);
2890
2891	if (xa->state == ATA_S_ONCHIP5)
2892	xa->state = ATA_S_COMPLETE2;
2893	#ifdef DIAGNOSTIC1
2894	else if (xa->state != ATA_S_ERROR3 && xa->state != ATA_S_TIMEOUT4)
2895	printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
2896	"slot %d\n", PORTNAME(ccb->ccb_port)(((ccb->ccb_port)->ap_sc)->sc_dev.dv_xname), xa->state,
2897	ccb->ccb_slot);
2898	#endif
2899	if (xa->state != ATA_S_TIMEOUT4)
2900	ata_complete(xa);
2901	}
2902
2903	void
2904	ahci_ata_cmd_timeout(void *arg)
2905	{
2906	struct ahci_ccb *ccb = arg;
2907	struct ata_xfer *xa = &ccb->ccb_xa;
2908	struct ahci_port *ap = ccb->ccb_port;
2909	int s, ccb_was_started, ncq_cmd;
2910	volatile u_int32_t *active;
2911
2912	s = splbio()splraise(0x3);
2913
2914	ncq_cmd = (xa->flags & ATA_F_NCQ(1<<6));
2915	active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
2916
2917	if (ccb->ccb_xa.state == ATA_S_PENDING1) {
2918	DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
2919	"before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
2920	TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry)do { if (((ccb)->ccb_entry.tqe_next) != ((void )0)) (ccb) ->ccb_entry.tqe_next->ccb_entry.tqe_prev = (ccb)->ccb_entry .tqe_prev; else (&ap->ap_ccb_pending)->tqh_last = ( ccb)->ccb_entry.tqe_prev; (ccb)->ccb_entry.tqe_prev = ( ccb)->ccb_entry.tqe_next; ((ccb)->ccb_entry.tqe_prev) = ((void )-1); ((ccb)->ccb_entry.tqe_next) = ((void )-1); } while (0);
2921	ccb_was_started = 0;
2922	} else if (ccb->ccb_xa.state == ATA_S_ONCHIP5 && ahci_port_intr(ap,
2923	1 << ccb->ccb_slot)) {
2924	DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
2925	"command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
2926	goto ret;
2927	} else if (ccb->ccb_xa.state != ATA_S_ONCHIP5) {
2928	DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
2929	"handled%s\n", PORTNAME(ap), ccb->ccb_slot,
2930	ISSET(*active, 1 << ccb->ccb_slot) ?
2931	" but slot is still active?" : ".");
2932	goto ret;
2933	} else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb ->ccb_slot))
2934	AHCI_PREG_CI), 1 << ccb->ccb_slot)((ahci_pread(ap, ncq_cmd ? 0x34 : 0x38)) & (1 << ccb ->ccb_slot)) && ISSET(active,((active) & (1 << ccb->ccb_slot))
2935	1 << ccb->ccb_slot)((*active) & (1 << ccb->ccb_slot))) {
2936	DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
2937	"IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
2938	ccb->ccb_slot);
2939	*active &= ~(1 << ccb->ccb_slot);
2940	ccb->ccb_done(ccb);
2941	goto ret;
2942	} else {
2943	ccb_was_started = 1;
2944	}
2945
2946	/* Complete the slot with a timeout error. */
2947	ccb->ccb_xa.state = ATA_S_TIMEOUT4;
2948	*active &= ~(1 << ccb->ccb_slot);
2949	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
2950	ccb->ccb_done(ccb); /* This won't issue pending commands or run the
2951	atascsi completion. */
2952
2953	/* Reset port to abort running command. */
2954	if (ccb_was_started) {
2955	DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
2956	"in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
2957	ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
2958	if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
2959	!= 0) {
2960	printf("%s: failed to reset port during timeout "
2961	"handling, disabling it\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
2962	ap->ap_state = AP_S_FATAL_ERROR4;
2963	}
2964
2965	/* Restart any other commands that were aborted by the reset. */
2966	if (*active) {
2967	DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
2968	"%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
2969	*active);
2970	if (ncq_cmd)
2971	ahci_pwrite(ap, AHCI_PREG_SACT0x34, *active);
2972	ahci_pwrite(ap, AHCI_PREG_CI0x38, *active);
2973	}
2974	}
2975
2976	/* Issue any pending commands now. */
2977	DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
2978	if (ccb_was_started)
2979	ahci_issue_pending_commands(ap, ncq_cmd);
2980	else if (ap->ap_active == 0)
2981	ahci_issue_pending_ncq_commands(ap);
2982
2983	/* Complete the timed out ata_xfer I/O (may generate new I/O). */
2984	DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
2985	ata_complete(xa);
2986
2987	DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
2988	ret:
2989	splx(s)spllower(s);
2990	}
2991
2992	void
2993	ahci_empty_done(struct ahci_ccb *ccb)
2994	{
2995	if (ccb->ccb_xa.state != ATA_S_ERROR3)
2996	ccb->ccb_xa.state = ATA_S_COMPLETE2;
2997	}
2998
2999	int
3000	ahci_pmp_read(struct ahci_port ap, int target, int which, u_int32_t datap)
3001	{
3002	struct ahci_ccb *ccb;
3003	struct ata_fis_h2d *fis;
3004	int error;
3005
3006	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3007	ccb->ccb_xa.flags = ATA_F_POLL(1<<3) \| ATA_F_GET_RFIS(1<<8);
3008	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f;
3009	ccb->ccb_xa.state = ATA_S_PENDING1;
3010
3011	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
3012	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3013	fis->type = ATA_FIS_TYPE_H2D0x27;
3014	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| SATA_PMP_CONTROL_PORT0x0f;
3015	fis->command = ATA_C_READ_PM0xe4;
3016	fis->features = which;
3017	fis->device = target \| ATA_H2D_DEVICE_LBA0x40;
3018	fis->control = ATA_FIS_CONTROL_4BIT0x08;
3019
3020	if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
3021	error = 1;
3022	} else {
3023	*datap = ccb->ccb_xa.rfis.sector_count \|
3024	(ccb->ccb_xa.rfis.lba_low << 8) \|
3025	(ccb->ccb_xa.rfis.lba_mid << 16) \|
3026	(ccb->ccb_xa.rfis.lba_high << 24);
3027	error = 0;
3028	}
3029	ahci_put_pmp_ccb(ccb);
3030	return (error);
3031	}
3032
3033	int
3034	ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
3035	{
3036	struct ahci_ccb *ccb;
3037	struct ata_fis_h2d *fis;
3038	int error;
3039
3040	ccb = ahci_get_pmp_ccb(ap); /* Always returns non-NULL. */
3041	ccb->ccb_xa.flags = ATA_F_POLL(1<<3);
3042	ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT0x0f;
3043	ccb->ccb_xa.state = ATA_S_PENDING1;
3044
3045	memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table))__builtin_memset((ccb->ccb_cmd_table), (0), (sizeof(struct ahci_cmd_table)));
3046	fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3047	fis->type = ATA_FIS_TYPE_H2D0x27;
3048	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| SATA_PMP_CONTROL_PORT0x0f;
3049	fis->command = ATA_C_WRITE_PM0xe8;
3050	fis->features = which;
3051	fis->device = target \| ATA_H2D_DEVICE_LBA0x40;
3052	fis->sector_count = (u_int8_t)data;
3053	fis->lba_low = (u_int8_t)(data >> 8);
3054	fis->lba_mid = (u_int8_t)(data >> 16);
3055	fis->lba_high = (u_int8_t)(data >> 24);
3056	fis->control = ATA_FIS_CONTROL_4BIT0x08;
3057
3058	error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
3059	ahci_put_pmp_ccb(ccb);
3060	return (error);
3061	}
3062
3063	int
3064	ahci_pmp_phy_status(struct ahci_port ap, int target, u_int32_t datap)
3065	{
3066	int error;
3067
3068	error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS0, datap);
3069	if (error == 0)
3070	error = ahci_pmp_write(ap, target, SATA_PMREG_SERR1, -1);
3071	if (error)
3072	*datap = 0;
3073
3074	return (error);
3075	}
3076
3077	int
3078	ahci_pmp_identify(struct ahci_port ap, int ret_nports)
3079	{
3080	u_int32_t chipid;
3081	u_int32_t rev;
3082	u_int32_t nports;
3083	u_int32_t features;
3084	u_int32_t enabled;
3085	int s;
3086
3087	s = splbio()splraise(0x3);
3088
3089	if (ahci_pmp_read(ap, 15, 0, &chipid) \|\|
3090	ahci_pmp_read(ap, 15, 1, &rev) \|\|
3091	ahci_pmp_read(ap, 15, 2, &nports) \|\|
3092	ahci_pmp_read(ap, 15, SATA_PMREG_FEA64, &features) \|\|
3093	ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN96, &enabled)) {
3094	printf("%s: port multiplier identification failed\n",
3095	PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname));
3096	splx(s)spllower(s);
3097	return (1);
3098	}
3099	splx(s)spllower(s);
3100
3101	nports &= 0x0F;
3102
3103	/* ignore SEMB port on SiI3726 port multiplier chips */
3104	if (chipid == 0x37261095) {
3105	nports--;
3106	}
3107
3108	printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
3109	"features: 0x%b, enabled: 0x%b\n", PORTNAME(ap)(((ap)->ap_sc)->sc_dev.dv_xname), chipid, rev,
3110	SATA_PFMT_PM_REV"\20" "\003PM1.1" "\002PM1.0", nports, features, SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST", enabled,
3111	SATA_PFMT_PM_FEA"\20" "\004AsyncNotify" "\003DynamicSSC" "\002PMREQ" "\001BIST");
3112
3113	*ret_nports = nports;
3114	return (0);
3115	}
3116
3117
3118	#ifdef HIBERNATE1
3119	void
3120	ahci_hibernate_io_start(struct ahci_port ap, struct ahci_ccb ccb)
3121	{
3122	ccb->ccb_cmd_hdr->prdbc = 0;
3123	ahci_pwrite(ap, AHCI_PREG_CI0x38, 1 << ccb->ccb_slot);
3124	}
3125
3126	int
3127	ahci_hibernate_io_poll(struct ahci_port ap, struct ahci_ccb ccb)
3128	{
3129	u_int32_t is, ci_saved;
3130	int process_error = 0;
3131
3132	is = ahci_pread(ap, AHCI_PREG_IS0x10);
3133
3134	ci_saved = ahci_pread(ap, AHCI_PREG_CI0x38);
3135
3136	if (is & AHCI_PREG_IS_DHRS(1<<0)) {
3137	u_int32_t tfd;
3138	u_int32_t cmd;
3139
3140	tfd = ahci_pread(ap, AHCI_PREG_TFD0x20);
3141	cmd = ahci_pread(ap, AHCI_PREG_CMD0x18);
3142	if ((tfd & AHCI_PREG_TFD_STS_ERR(1<<0)) &&
3143	(cmd & AHCI_PREG_CMD_CR(1<<15)) == 0) {
3144	process_error = 1;
3145	} else {
3146	ahci_pwrite(ap, AHCI_PREG_IS0x10, AHCI_PREG_IS_DHRS(1<<0));
3147	}
3148	} else if (is & (AHCI_PREG_IS_TFES(1<<30) \| AHCI_PREG_IS_HBFS(1<<29) \|
3149	AHCI_PREG_IS_IFS(1<<27) \| AHCI_PREG_IS_OFS(1<<24) \| AHCI_PREG_IS_UFS(1<<4))) {
3150	process_error = 1;
3151	}
3152
3153	/* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
3154	if (process_error) {
3155
3156	/* Turn off ST to clear CI and SACT. */
3157	ahci_port_stop(ap, 0);
3158
3159	/* just return an error indicator? we can't meaningfully
3160	* recover, and on the way back out we'll DVACT_RESUME which
3161	* resets and reinits the port.
3162	*/
3163	return (EIO5);
3164	}
3165
3166	/* command is finished when the bit in CI for the slot goes to 0 */
3167	if (ci_saved & (1 << ccb->ccb_slot)) {
3168	return (EAGAIN35);
3169	}
3170
3171	return (0);
3172	}
3173
3174	void
3175	ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
3176	{
3177	struct ata_xfer *xa = &ccb->ccb_xa;
3178	struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
3179	struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
3180	int i;
3181	paddr_t data_phys;
3182	u_int64_t data_bus_phys;
3183	vaddr_t data_addr;
3184	size_t seglen;
3185	size_t buflen;
3186
3187	if (xa->datalen == 0) {
3188	ccb->ccb_cmd_hdr->prdtl = 0;
3189	return;
3190	}
3191
3192	/* derived from i386/amd64 _bus_dma_load_buffer;
3193	* for amd64 the buffer will always be dma safe.
3194	*/
3195
3196	buflen = xa->datalen;
3197	data_addr = (vaddr_t)xa->data;
3198	for (i = 0; buflen > 0; i++) {
3199	pmap_extract(pmap_kernel()(&kernel_pmap_store), data_addr, &data_phys);
3200	data_bus_phys = data_phys;
3201
3202	seglen = PAGE_SIZE(1 << 12) - ((u_long)data_addr & PGOFSET((1 << 12) - 1));
3203	if (buflen < seglen)
3204	seglen = buflen;
3205
3206	ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
3207
3208	data_addr += seglen;
3209	buflen -= seglen;
3210	}
3211
3212	htolem16(&cmd_slot->prdtl, i)((__uint16_t )(&cmd_slot->prdtl) = ((__uint16_t)(i)) );
3213	}
3214
3215	int
3216	ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
3217	int op, void *page)
3218	{
3219	/* we use the 'real' ahci_port and ahci_softc here, but
3220	* never write to them
3221	*/
3222	struct {
3223	struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */
3224	struct ahci_rfis rfis; /* 1k aligned, 256 bytes */
3225	/* cmd table isn't actually used because of mysteries */
3226	struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */
3227	struct ahci_port *ap;
3228	struct ahci_ccb ccb_buf;
3229	struct ahci_ccb *ccb;
3230	struct ahci_cmd_hdr *hdr_buf;
3231	int pmp_port;
3232	daddr_t poffset;
3233	size_t psize;
3234	} *my = page;
3235	struct ata_fis_h2d *fis;
3236	u_int32_t sector_count;
3237	struct ahci_cmd_hdr *cmd_slot;
3238	int rc;
3239	int timeout;
3240	u_int16_t flags;
3241
3242	if (op == HIB_INIT-1) {
3243	struct device *disk;
3244	struct device *scsibus;
3245	struct ahci_softc *sc;
3246	extern struct cfdriver sd_cd;
3247	struct scsi_link *link;
3248	struct scsibus_softc *bus_sc;
3249	int port;
3250	paddr_t page_phys;
3251	u_int64_t item_phys;
3252	u_int32_t cmd;
3253
3254	my->poffset = blkno;
3255	my->psize = size;
3256
3257	/* map dev to an ahci port */
3258	disk = disk_lookup(&sd_cd, DISKUNIT(dev)(((unsigned)((dev) & 0xff) \| (((dev) & 0xffff0000) >> 8)) / 16));
3259	scsibus = disk->dv_parent;
3260	sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
3261
3262	/* find the scsi_link for the device, which has the port */
3263	port = -1;
3264	bus_sc = (struct scsibus_softc *)scsibus;
3265	SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list)for((link) = ((&bus_sc->sc_link_list)->slh_first); ( link) != ((void *)0); (link) = ((link)->bus_list.sle_next) ) {
3266	if (link->device_softc == disk) {
3267	port = link->target;
3268	if (link->lun > 0)
3269	my->pmp_port = link->lun - 1;
3270	else
3271	my->pmp_port = 0;
3272
3273	break;
3274	}
3275	}
3276	if (port == -1) {
3277	/* don't know where the disk is */
3278	return (EIO5);
3279	}
3280
3281	my->ap = sc->sc_ports[port];
3282
3283	/* we're going to use the first command slot,
3284	* so ensure it's not already in use
3285	*/
3286	if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT6) {
3287	/* this shouldn't happen, we should be idle */
3288	return (EIO5);
3289	}
3290
3291	/* stop the port so we can relocate to the hibernate page */
3292	if (ahci_port_stop(my->ap, 1)) {
3293	return (EIO5);
3294	}
3295	ahci_pwrite(my->ap, AHCI_PREG_SCTL0x2c, 0);
3296
3297	pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)page, &page_phys);
3298
3299	/* Setup RFIS base address */
3300	item_phys = page_phys + ((void *)&my->rfis - page);
3301	ahci_pwrite(my->ap, AHCI_PREG_FBU0x0c,
3302	(u_int32_t)(item_phys >> 32));
3303	ahci_pwrite(my->ap, AHCI_PREG_FB0x08, (u_int32_t)item_phys);
3304
3305	/* Enable FIS reception and activate port. */
3306	cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
3307	cmd \|= AHCI_PREG_CMD_FRE(1<<4) \| AHCI_PREG_CMD_POD(1<<2) \|
3308	AHCI_PREG_CMD_SUD(1<<1);
3309	ahci_pwrite(my->ap, AHCI_PREG_CMD0x18, cmd \|
3310	AHCI_PREG_CMD_ICC_ACTIVE0x10000000);
3311
3312	/* Check whether port activated. */
3313	cmd = ahci_pread(my->ap, AHCI_PREG_CMD0x18) & ~AHCI_PREG_CMD_ICC0xf0000000;
3314	if (!ISSET(cmd, AHCI_PREG_CMD_FRE)((cmd) & ((1<<4)))) {
3315	return (EIO5);
3316	}
3317
3318	/* Set up the single CCB */
3319	my->ccb = &my->ccb_buf;
3320	my->ccb->ccb_slot = 0;
3321	my->ccb->ccb_port = my->ap;
3322
3323	/* Setup command list base address */
3324	item_phys = page_phys + ((void *)&my->cmd_hdr - page);
3325	ahci_pwrite(my->ap, AHCI_PREG_CLBU0x04,
3326	(u_int32_t)(item_phys >> 32));
3327	ahci_pwrite(my->ap, AHCI_PREG_CLB0x00, (u_int32_t)item_phys);
3328
3329	my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
3330
3331	/* use existing cmd table - moving to a new one fails */
3332	my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
3333	pmap_extract(pmap_kernel()(&kernel_pmap_store),
3334	(vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table)((void *)(my->ap->ap_dmamem_cmd_table)->adm_kva),
3335	&page_phys);
3336	item_phys = page_phys;
3337	#if 0
3338	/* use cmd table in hibernate page (doesn't work) */
3339	my->ccb->ccb_cmd_table = &my->cmd_table;
3340	item_phys = page_phys + ((void *)&my->cmd_table - page);
3341	#endif
3342	htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys)((__uint64_t )(&my->ccb->ccb_cmd_hdr->ctba) = ( (__uint64_t)(item_phys)));
3343
3344	my->ccb->ccb_xa.fis =
3345	(struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
3346	my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
3347	my->ccb->ccb_xa.tag = 0;
3348
3349	/* Wait for ICC change to complete */
3350	ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1)ahci_pwait_eq((my->ap), (0x18), (0xf0000000), 0, (1));
3351
3352	if (ahci_port_start(my->ap, 0)((my->ap)->ap_sc->sc_port_start((my->ap), (0)))) {
3353	return (EIO5);
3354	}
3355
3356	/* Flush interrupts for port */
3357	ahci_pwrite(my->ap, AHCI_PREG_IS0x10, ahci_pread(my->ap,
3358	AHCI_PREG_IS0x10));
3359	ahci_write(sc, AHCI_REG_IS0x008, 1 << port);
3360
3361	ahci_enable_interrupts(my->ap);
3362	return (0);
3363	} else if (op == HIB_DONE-2) {
3364	ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME4);
3365	return (0);
3366	}
3367
3368	if (blkno > my->psize)
3369	return (E2BIG7);
3370	blkno += my->poffset;
3371
3372	/* build fis */
3373	sector_count = size / 512; /* dlg promises this is okay */
3374	my->ccb->ccb_xa.flags = op == HIB_W1 ? ATA_F_WRITE(1<<1) : ATA_F_READ(1<<0);
3375	fis = my->ccb->ccb_xa.fis;
3376	fis->flags = ATA_H2D_FLAGS_CMD(1<<7) \| my->pmp_port;
3377	fis->lba_low = blkno & 0xff;
3378	fis->lba_mid = (blkno >> 8) & 0xff;
3379	fis->lba_high = (blkno >> 16) & 0xff;
3380
3381	if (sector_count > 0x100 \|\| blkno > 0xfffffff) {
3382	/* Use LBA48 */
3383	fis->command = op == HIB_W1 ? ATA_C_WRITEDMA_EXT0x35 :
3384	ATA_C_READDMA_EXT0x25;
3385	fis->device = ATA_H2D_DEVICE_LBA0x40;
3386	fis->lba_low_exp = (blkno >> 24) & 0xff;
3387	fis->lba_mid_exp = (blkno >> 32) & 0xff;
3388	fis->lba_high_exp = (blkno >> 40) & 0xff;
3389	fis->sector_count = sector_count & 0xff;
3390	fis->sector_count_exp = (sector_count >> 8) & 0xff;
3391	} else {
3392	/* Use LBA */
3393	fis->command = op == HIB_W1 ? ATA_C_WRITEDMA0xca : ATA_C_READDMA0xc8;
3394	fis->device = ATA_H2D_DEVICE_LBA0x40 \| ((blkno >> 24) & 0x0f);
3395	fis->sector_count = sector_count & 0xff;
3396	}
3397
3398	my->ccb->ccb_xa.data = (void *)addr;
3399	my->ccb->ccb_xa.datalen = size;
3400	my->ccb->ccb_xa.pmp_port = my->pmp_port;
3401	my->ccb->ccb_xa.flags \|= ATA_F_POLL(1<<3);
3402
3403	cmd_slot = my->ccb->ccb_cmd_hdr;
3404	flags = 5; /* FIS length (in DWORDs) */
3405	flags \|= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT12;
3406
3407	if (op == HIB_W1)
3408	flags \|= AHCI_CMD_LIST_FLAG_W(1<<6);
3409
3410	htolem16(&cmd_slot->flags, flags)((__uint16_t )(&cmd_slot->flags) = ((__uint16_t)(flags )));
3411
3412	ahci_hibernate_load_prdt(my->ccb);
3413
3414	ahci_hibernate_io_start(my->ap, my->ccb);
3415	timeout = 1000000;
3416	while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN35) {
3417	delay(1)(*delay_func)(1);
3418	timeout--;
3419	if (timeout == 0) {
3420	return (EIO5);
3421	}
3422	}
3423
3424	return (0);
3425	}
3426
3427	#endif