Bug Summary

File:dev/usb/xhci.c
Warning:line 2422, column 54
Access to field 'bDescriptorType' results in a dereference of a null pointer (loaded from variable 'buf')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name xhci.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/usb/xhci.c
1/* $OpenBSD: xhci.c,v 1.130 2023/07/20 09:43:00 claudio Exp $ */
2
3/*
4 * Copyright (c) 2014-2015 Martin Pieuchot
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/kernel.h>
22#include <sys/malloc.h>
23#include <sys/device.h>
24#include <sys/queue.h>
25#include <sys/timeout.h>
26#include <sys/pool.h>
27#include <sys/endian.h>
28#include <sys/rwlock.h>
29
30#include <machine/bus.h>
31
32#include <dev/usb/usb.h>
33#include <dev/usb/usbdi.h>
34#include <dev/usb/usbdivar.h>
35#include <dev/usb/usb_mem.h>
36
37#include <dev/usb/xhcireg.h>
38#include <dev/usb/xhcivar.h>
39
40struct cfdriver xhci_cd = {
41 NULL((void *)0), "xhci", DV_DULL, CD_SKIPHIBERNATE2
42};
43
44#ifdef XHCI_DEBUG
45#define DPRINTF(x) do { if (xhcidebug) printf x; } while(0)
46#define DPRINTFN(n,x) do { if (xhcidebug>(n)) printf x; } while (0)
47int xhcidebug = 3;
48#else
49#define DPRINTF(x)
50#define DPRINTFN(n,x)
51#endif
52
53#define DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname) ((sc)->sc_bus.bdev.dv_xname)
54
55#define TRBOFF(r, trb)((char *)(trb) - (char *)((r)->trbs)) ((char *)(trb) - (char *)((r)->trbs))
56#define DEQPTR(r)((r).dma.paddr + (sizeof(struct xhci_trb) * (r).index)) ((r).dma.paddr + (sizeof(struct xhci_trb) * (r).index))
57
58struct pool *xhcixfer;
59
60struct xhci_pipe {
61 struct usbd_pipe pipe;
62
63 uint8_t dci;
64 uint8_t slot; /* Device slot ID */
65 struct xhci_ring ring;
66
67 /*
68 * XXX used to pass the xfer pointer back to the
69 * interrupt routine, better way?
70 */
71 struct usbd_xfer *pending_xfers[XHCI_MAX_XFER(16 * 16)];
72 struct usbd_xfer *aborted_xfer;
73 int halted;
74 size_t free_trbs;
75 int skip;
76#define TRB_PROCESSED_NO0 0
77#define TRB_PROCESSED_YES1 1
78#define TRB_PROCESSED_SHORT2 2
79 uint8_t trb_processed[XHCI_MAX_XFER(16 * 16)];
80};
81
82int xhci_reset(struct xhci_softc *);
83int xhci_intr1(struct xhci_softc *);
84void xhci_event_dequeue(struct xhci_softc *);
85void xhci_event_xfer(struct xhci_softc *, uint64_t, uint32_t, uint32_t);
86int xhci_event_xfer_generic(struct xhci_softc *, struct usbd_xfer *,
87 struct xhci_pipe *, uint32_t, int, uint8_t, uint8_t, uint8_t);
88int xhci_event_xfer_isoc(struct usbd_xfer *, struct xhci_pipe *,
89 uint32_t, int, uint8_t);
90void xhci_event_command(struct xhci_softc *, uint64_t);
91void xhci_event_port_change(struct xhci_softc *, uint64_t, uint32_t);
92int xhci_pipe_init(struct xhci_softc *, struct usbd_pipe *);
93int xhci_context_setup(struct xhci_softc *, struct usbd_pipe *);
94int xhci_scratchpad_alloc(struct xhci_softc *, int);
95void xhci_scratchpad_free(struct xhci_softc *);
96int xhci_softdev_alloc(struct xhci_softc *, uint8_t);
97void xhci_softdev_free(struct xhci_softc *, uint8_t);
98int xhci_ring_alloc(struct xhci_softc *, struct xhci_ring *, size_t,
99 size_t);
100void xhci_ring_free(struct xhci_softc *, struct xhci_ring *);
101void xhci_ring_reset(struct xhci_softc *, struct xhci_ring *);
102struct xhci_trb *xhci_ring_consume(struct xhci_softc *, struct xhci_ring *);
103struct xhci_trb *xhci_ring_produce(struct xhci_softc *, struct xhci_ring *);
104
105struct xhci_trb *xhci_xfer_get_trb(struct xhci_softc *, struct usbd_xfer*,
106 uint8_t *, int);
107void xhci_xfer_done(struct usbd_xfer *xfer);
108/* xHCI command helpers. */
109int xhci_command_submit(struct xhci_softc *, struct xhci_trb *, int);
110int xhci_command_abort(struct xhci_softc *);
111
112void xhci_cmd_reset_ep_async(struct xhci_softc *, uint8_t, uint8_t);
113void xhci_cmd_set_tr_deq_async(struct xhci_softc *, uint8_t, uint8_t, uint64_t);
114int xhci_cmd_configure_ep(struct xhci_softc *, uint8_t, uint64_t);
115int xhci_cmd_stop_ep(struct xhci_softc *, uint8_t, uint8_t);
116int xhci_cmd_slot_control(struct xhci_softc *, uint8_t *, int);
117int xhci_cmd_set_address(struct xhci_softc *, uint8_t, uint64_t, uint32_t);
118#ifdef XHCI_DEBUG
119int xhci_cmd_noop(struct xhci_softc *);
120#endif
121
122/* XXX should be part of the Bus interface. */
123void xhci_abort_xfer(struct usbd_xfer *, usbd_status);
124void xhci_pipe_close(struct usbd_pipe *);
125void xhci_noop(struct usbd_xfer *);
126
127void xhci_timeout(void *);
128void xhci_timeout_task(void *);
129
130/* USBD Bus Interface. */
131usbd_status xhci_pipe_open(struct usbd_pipe *);
132int xhci_setaddr(struct usbd_device *, int);
133void xhci_softintr(void *);
134void xhci_poll(struct usbd_bus *);
135struct usbd_xfer *xhci_allocx(struct usbd_bus *);
136void xhci_freex(struct usbd_bus *, struct usbd_xfer *);
137
138usbd_status xhci_root_ctrl_transfer(struct usbd_xfer *);
139usbd_status xhci_root_ctrl_start(struct usbd_xfer *);
140
141usbd_status xhci_root_intr_transfer(struct usbd_xfer *);
142usbd_status xhci_root_intr_start(struct usbd_xfer *);
143void xhci_root_intr_abort(struct usbd_xfer *);
144void xhci_root_intr_done(struct usbd_xfer *);
145
146usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *);
147usbd_status xhci_device_ctrl_start(struct usbd_xfer *);
148void xhci_device_ctrl_abort(struct usbd_xfer *);
149
150usbd_status xhci_device_generic_transfer(struct usbd_xfer *);
151usbd_status xhci_device_generic_start(struct usbd_xfer *);
152void xhci_device_generic_abort(struct usbd_xfer *);
153void xhci_device_generic_done(struct usbd_xfer *);
154
155usbd_status xhci_device_isoc_transfer(struct usbd_xfer *);
156usbd_status xhci_device_isoc_start(struct usbd_xfer *);
157
158#define XHCI_INTR_ENDPT1 1
159
160const struct usbd_bus_methods xhci_bus_methods = {
161 .open_pipe = xhci_pipe_open,
162 .dev_setaddr = xhci_setaddr,
163 .soft_intr = xhci_softintr,
164 .do_poll = xhci_poll,
165 .allocx = xhci_allocx,
166 .freex = xhci_freex,
167};
168
169const struct usbd_pipe_methods xhci_root_ctrl_methods = {
170 .transfer = xhci_root_ctrl_transfer,
171 .start = xhci_root_ctrl_start,
172 .abort = xhci_noop,
173 .close = xhci_pipe_close,
174 .done = xhci_noop,
175};
176
177const struct usbd_pipe_methods xhci_root_intr_methods = {
178 .transfer = xhci_root_intr_transfer,
179 .start = xhci_root_intr_start,
180 .abort = xhci_root_intr_abort,
181 .close = xhci_pipe_close,
182 .done = xhci_root_intr_done,
183};
184
185const struct usbd_pipe_methods xhci_device_ctrl_methods = {
186 .transfer = xhci_device_ctrl_transfer,
187 .start = xhci_device_ctrl_start,
188 .abort = xhci_device_ctrl_abort,
189 .close = xhci_pipe_close,
190 .done = xhci_noop,
191};
192
193const struct usbd_pipe_methods xhci_device_intr_methods = {
194 .transfer = xhci_device_generic_transfer,
195 .start = xhci_device_generic_start,
196 .abort = xhci_device_generic_abort,
197 .close = xhci_pipe_close,
198 .done = xhci_device_generic_done,
199};
200
201const struct usbd_pipe_methods xhci_device_bulk_methods = {
202 .transfer = xhci_device_generic_transfer,
203 .start = xhci_device_generic_start,
204 .abort = xhci_device_generic_abort,
205 .close = xhci_pipe_close,
206 .done = xhci_device_generic_done,
207};
208
209const struct usbd_pipe_methods xhci_device_isoc_methods = {
210 .transfer = xhci_device_isoc_transfer,
211 .start = xhci_device_isoc_start,
212 .abort = xhci_device_generic_abort,
213 .close = xhci_pipe_close,
214 .done = xhci_noop,
215};
216
217#ifdef XHCI_DEBUG
218static void
219xhci_dump_trb(struct xhci_trb *trb)
220{
221 printf("trb=%p (0x%016llx 0x%08x 0x%b)\n", trb,
222 (long long)letoh64(trb->trb_paddr)((__uint64_t)(trb->trb_paddr)), letoh32(trb->trb_status)((__uint32_t)(trb->trb_status)),
223 (int)letoh32(trb->trb_flags)((__uint32_t)(trb->trb_flags)), XHCI_TRB_FLAGS_BITMASK"\20" "\040SIA" "\022TRT_OUT" "\021DIR_IN" "\012BSR" "\007IDT"
"\006IOC" "\005CHAIN" "\004NOSNOOP" "\003ISP" "\002LINKSEG" "\001CYCLE");
224}
225#endif
226
227int usbd_dma_contig_alloc(struct usbd_bus *, struct usbd_dma_info *,
228 void **, bus_size_t, bus_size_t, bus_size_t);
229void usbd_dma_contig_free(struct usbd_bus *, struct usbd_dma_info *);
230
231int
232usbd_dma_contig_alloc(struct usbd_bus *bus, struct usbd_dma_info *dma,
233 void **kvap, bus_size_t size, bus_size_t alignment, bus_size_t boundary)
234{
235 int error;
236
237 dma->tag = bus->dmatag;
238 dma->size = size;
239
240 error = bus_dmamap_create(dma->tag, size, 1, size, boundary,(*(dma->tag)->_dmamap_create)((dma->tag), (size), (1
), (size), (boundary), (0x0001), (&dma->map))
241 BUS_DMA_NOWAIT, &dma->map)(*(dma->tag)->_dmamap_create)((dma->tag), (size), (1
), (size), (boundary), (0x0001), (&dma->map));
242 if (error != 0)
243 return (error);
244
245 error = bus_dmamem_alloc(dma->tag, size, alignment, boundary, &dma->seg,(*(dma->tag)->_dmamem_alloc)((dma->tag), (size), (alignment
), (boundary), (&dma->seg), (1), (&dma->nsegs),
(0x0001 | 0x1000))
246 1, &dma->nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO)(*(dma->tag)->_dmamem_alloc)((dma->tag), (size), (alignment
), (boundary), (&dma->seg), (1), (&dma->nsegs),
(0x0001 | 0x1000));
247 if (error != 0)
248 goto destroy;
249
250 error = bus_dmamem_map(dma->tag, &dma->seg, 1, size, &dma->vaddr,(*(dma->tag)->_dmamem_map)((dma->tag), (&dma->
seg), (1), (size), (&dma->vaddr), (0x0001 | 0x0004))
251 BUS_DMA_NOWAIT | BUS_DMA_COHERENT)(*(dma->tag)->_dmamem_map)((dma->tag), (&dma->
seg), (1), (size), (&dma->vaddr), (0x0001 | 0x0004));
252 if (error != 0)
253 goto free;
254
255 error = bus_dmamap_load_raw(dma->tag, dma->map, &dma->seg, 1, size,(*(dma->tag)->_dmamap_load_raw)((dma->tag), (dma->
map), (&dma->seg), (1), (size), (0x0001))
256 BUS_DMA_NOWAIT)(*(dma->tag)->_dmamap_load_raw)((dma->tag), (dma->
map), (&dma->seg), (1), (size), (0x0001));
257 if (error != 0)
258 goto unmap;
259
260 bus_dmamap_sync(dma->tag, dma->map, 0, size, BUS_DMASYNC_PREREAD |(*(dma->tag)->_dmamap_sync)((dma->tag), (dma->map
), (0), (size), (0x01 | 0x04))
261 BUS_DMASYNC_PREWRITE)(*(dma->tag)->_dmamap_sync)((dma->tag), (dma->map
), (0), (size), (0x01 | 0x04));
262
263 dma->paddr = dma->map->dm_segs[0].ds_addr;
264 if (kvap != NULL((void *)0))
265 *kvap = dma->vaddr;
266
267 return (0);
268
269unmap:
270 bus_dmamem_unmap(dma->tag, dma->vaddr, size)(*(dma->tag)->_dmamem_unmap)((dma->tag), (dma->vaddr
), (size));
271free:
272 bus_dmamem_free(dma->tag, &dma->seg, 1)(*(dma->tag)->_dmamem_free)((dma->tag), (&dma->
seg), (1));
273destroy:
274 bus_dmamap_destroy(dma->tag, dma->map)(*(dma->tag)->_dmamap_destroy)((dma->tag), (dma->
map));
275 return (error);
276}
277
278void
279usbd_dma_contig_free(struct usbd_bus *bus, struct usbd_dma_info *dma)
280{
281 if (dma->map != NULL((void *)0)) {
282 bus_dmamap_sync(bus->dmatag, dma->map, 0, dma->size,(*(bus->dmatag)->_dmamap_sync)((bus->dmatag), (dma->
map), (0), (dma->size), (0x02 | 0x08))
283 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)(*(bus->dmatag)->_dmamap_sync)((bus->dmatag), (dma->
map), (0), (dma->size), (0x02 | 0x08));
284 bus_dmamap_unload(bus->dmatag, dma->map)(*(bus->dmatag)->_dmamap_unload)((bus->dmatag), (dma
->map));
285 bus_dmamem_unmap(bus->dmatag, dma->vaddr, dma->size)(*(bus->dmatag)->_dmamem_unmap)((bus->dmatag), (dma->
vaddr), (dma->size));
286 bus_dmamem_free(bus->dmatag, &dma->seg, 1)(*(bus->dmatag)->_dmamem_free)((bus->dmatag), (&
dma->seg), (1));
287 bus_dmamap_destroy(bus->dmatag, dma->map)(*(bus->dmatag)->_dmamap_destroy)((bus->dmatag), (dma
->map));
288 dma->map = NULL((void *)0);
289 }
290}
291
292int
293xhci_init(struct xhci_softc *sc)
294{
295 uint32_t hcr;
296 int npage, error;
297
298 sc->sc_bus.usbrev = USBREV_3_05;
299 sc->sc_bus.methods = &xhci_bus_methods;
300 sc->sc_bus.pipe_size = sizeof(struct xhci_pipe);
301
302 sc->sc_oper_off = XREAD1(sc, XHCI_CAPLENGTH)(((sc)->iot)->read_1(((sc)->ioh), ((0x00))));
303 sc->sc_door_off = XREAD4(sc, XHCI_DBOFF)(((sc)->iot)->read_4(((sc)->ioh), ((0x14))));
304 sc->sc_runt_off = XREAD4(sc, XHCI_RTSOFF)(((sc)->iot)->read_4(((sc)->ioh), ((0x18))));
305
306 sc->sc_version = XREAD2(sc, XHCI_HCIVERSION)(((sc)->iot)->read_2(((sc)->ioh), ((0x02))));
307 printf(", xHCI %x.%x\n", sc->sc_version >> 8, sc->sc_version & 0xff);
308
309#ifdef XHCI_DEBUG
310 printf("%s: CAPLENGTH=%#lx\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname), sc->sc_oper_off);
311 printf("%s: DOORBELL=%#lx\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname), sc->sc_door_off);
312 printf("%s: RUNTIME=%#lx\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname), sc->sc_runt_off);
313#endif
314
315 error = xhci_reset(sc);
316 if (error)
317 return (error);
318
319 if (xhcixfer == NULL((void *)0)) {
320 xhcixfer = malloc(sizeof(struct pool), M_USBHC103, M_NOWAIT0x0002);
321 if (xhcixfer == NULL((void *)0)) {
322 printf("%s: unable to allocate pool descriptor\n",
323 DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
324 return (ENOMEM12);
325 }
326 pool_init(xhcixfer, sizeof(struct xhci_xfer), 0, IPL_SOFTUSB0x2,
327 0, "xhcixfer", NULL((void *)0));
328 }
329
330 hcr = XREAD4(sc, XHCI_HCCPARAMS)(((sc)->iot)->read_4(((sc)->ioh), ((0x10))));
331 sc->sc_ctxsize = XHCI_HCC_CSZ(hcr)(((hcr) >> 2) & 0x1) ? 64 : 32;
332 DPRINTF(("%s: %d bytes context\n", DEVNAME(sc), sc->sc_ctxsize));
333
334#ifdef XHCI_DEBUG
335 hcr = XOREAD4(sc, XHCI_PAGESIZE)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x08))));
336 printf("%s: supported page size 0x%08x\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname), hcr);
337#endif
338 /* Use 4K for the moment since it's easier. */
339 sc->sc_pagesize = 4096;
340
341 /* Get port and device slot numbers. */
342 hcr = XREAD4(sc, XHCI_HCSPARAMS1)(((sc)->iot)->read_4(((sc)->ioh), ((0x04))));
343 sc->sc_noport = XHCI_HCS1_N_PORTS(hcr)(((hcr) >> 24) & 0xff);
344 sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(hcr)((hcr) & 0xff);
345 DPRINTF(("%s: %d ports and %d slots\n", DEVNAME(sc), sc->sc_noport,
346 sc->sc_noslot));
347
348 /* Setup Device Context Base Address Array. */
349 error = usbd_dma_contig_alloc(&sc->sc_bus, &sc->sc_dcbaa.dma,
350 (void **)&sc->sc_dcbaa.segs, (sc->sc_noslot + 1) * sizeof(uint64_t),
351 XHCI_DCBAA_ALIGN64, sc->sc_pagesize);
352 if (error)
353 return (ENOMEM12);
354
355 /* Setup command ring. */
356 rw_init(&sc->sc_cmd_lock, "xhcicmd")_rw_init_flags(&sc->sc_cmd_lock, "xhcicmd", 0, ((void *
)0));
357 error = xhci_ring_alloc(sc, &sc->sc_cmd_ring, XHCI_MAX_CMDS(16 * 1),
358 XHCI_CMDS_RING_ALIGN64);
359 if (error) {
360 printf("%s: could not allocate command ring.\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
361 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_dcbaa.dma);
362 return (error);
363 }
364
365 /* Setup one event ring and its segment table (ERST). */
366 error = xhci_ring_alloc(sc, &sc->sc_evt_ring, XHCI_MAX_EVTS(16 * 13),
367 XHCI_EVTS_RING_ALIGN64);
368 if (error) {
369 printf("%s: could not allocate event ring.\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
370 xhci_ring_free(sc, &sc->sc_cmd_ring);
371 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_dcbaa.dma);
372 return (error);
373 }
374
375 /* Allocate the required entry for the segment table. */
376 error = usbd_dma_contig_alloc(&sc->sc_bus, &sc->sc_erst.dma,
377 (void **)&sc->sc_erst.segs, sizeof(struct xhci_erseg),
378 XHCI_ERST_ALIGN64, XHCI_ERST_BOUNDARY0);
379 if (error) {
380 printf("%s: could not allocate segment table.\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
381 xhci_ring_free(sc, &sc->sc_evt_ring);
382 xhci_ring_free(sc, &sc->sc_cmd_ring);
383 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_dcbaa.dma);
384 return (ENOMEM12);
385 }
386
387 /* Set our ring address and size in its corresponding segment. */
388 sc->sc_erst.segs[0].er_addr = htole64(sc->sc_evt_ring.dma.paddr)((__uint64_t)(sc->sc_evt_ring.dma.paddr));
389 sc->sc_erst.segs[0].er_size = htole32(XHCI_MAX_EVTS)((__uint32_t)((16 * 13)));
390 sc->sc_erst.segs[0].er_rsvd = 0;
391 bus_dmamap_sync(sc->sc_erst.dma.tag, sc->sc_erst.dma.map, 0,(*(sc->sc_erst.dma.tag)->_dmamap_sync)((sc->sc_erst.
dma.tag), (sc->sc_erst.dma.map), (0), (sc->sc_erst.dma.
size), (0x01 | 0x04))
392 sc->sc_erst.dma.size, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(sc->sc_erst.dma.tag)->_dmamap_sync)((sc->sc_erst.
dma.tag), (sc->sc_erst.dma.map), (0), (sc->sc_erst.dma.
size), (0x01 | 0x04));
393
394 /* Get the number of scratch pages and configure them if necessary. */
395 hcr = XREAD4(sc, XHCI_HCSPARAMS2)(((sc)->iot)->read_4(((sc)->ioh), ((0x08))));
396 npage = XHCI_HCS2_SPB_MAX(hcr)((((hcr) >> 16) & 0x3e0) | (((hcr) >> 27) &
0x1f));
397 DPRINTF(("%s: %u scratch pages, ETE=%u, IST=0x%x\n", DEVNAME(sc), npage,
398 XHCI_HCS2_ETE(hcr), XHCI_HCS2_IST(hcr)));
399
400 if (npage > 0 && xhci_scratchpad_alloc(sc, npage)) {
401 printf("%s: could not allocate scratchpad.\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
402 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_erst.dma);
403 xhci_ring_free(sc, &sc->sc_evt_ring);
404 xhci_ring_free(sc, &sc->sc_cmd_ring);
405 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_dcbaa.dma);
406 return (ENOMEM12);
407 }
408
409
410 return (0);
411}
412
413void
414xhci_config(struct xhci_softc *sc)
415{
416 uint64_t paddr;
417 uint32_t hcr;
418
419 /* Make sure to program a number of device slots we can handle. */
420 if (sc->sc_noslot > USB_MAX_DEVICES128)
421 sc->sc_noslot = USB_MAX_DEVICES128;
422 hcr = XOREAD4(sc, XHCI_CONFIG)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x38)))) & ~XHCI_CONFIG_SLOTS_MASK0x000000ff;
423 XOWRITE4(sc, XHCI_CONFIG, hcr | sc->sc_noslot)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x38)), ((hcr | sc->sc_noslot))));
424
425 /* Set the device context base array address. */
426 paddr = (uint64_t)sc->sc_dcbaa.dma.paddr;
427 XOWRITE4(sc, XHCI_DCBAAP_LO, (uint32_t)paddr)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x30)), (((uint32_t)paddr))));
428 XOWRITE4(sc, XHCI_DCBAAP_HI, (uint32_t)(paddr >> 32))(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x34)), (((uint32_t)(paddr >> 32)))));
429
430 DPRINTF(("%s: DCBAAP=%#x%#x\n", DEVNAME(sc),
431 XOREAD4(sc, XHCI_DCBAAP_HI), XOREAD4(sc, XHCI_DCBAAP_LO)));
432
433 /* Set the command ring address. */
434 paddr = (uint64_t)sc->sc_cmd_ring.dma.paddr;
435 XOWRITE4(sc, XHCI_CRCR_LO, ((uint32_t)paddr) | XHCI_CRCR_LO_RCS)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x18)), ((((uint32_t)paddr) | 0x00000001))));
436 XOWRITE4(sc, XHCI_CRCR_HI, (uint32_t)(paddr >> 32))(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x1C)), (((uint32_t)(paddr >> 32)))));
437
438 DPRINTF(("%s: CRCR=%#x%#x (%016llx)\n", DEVNAME(sc),
439 XOREAD4(sc, XHCI_CRCR_HI), XOREAD4(sc, XHCI_CRCR_LO), paddr));
440
441 /* Set the ERST count number to 1, since we use only one event ring. */
442 XRWRITE4(sc, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1))(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0028 + (0x20 * (0))))), ((((1) & 0xffff)))));
443
444 /* Set the segment table address. */
445 paddr = (uint64_t)sc->sc_erst.dma.paddr;
446 XRWRITE4(sc, XHCI_ERSTBA_LO(0), (uint32_t)paddr)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0030 + (0x20 * (0))))), (((uint32_t)paddr))));
447 XRWRITE4(sc, XHCI_ERSTBA_HI(0), (uint32_t)(paddr >> 32))(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0034 + (0x20 * (0))))), (((uint32_t)(paddr >> 32
)))));
448
449 DPRINTF(("%s: ERSTBA=%#x%#x\n", DEVNAME(sc),
450 XRREAD4(sc, XHCI_ERSTBA_HI(0)), XRREAD4(sc, XHCI_ERSTBA_LO(0))));
451
452 /* Set the ring dequeue address. */
453 paddr = (uint64_t)sc->sc_evt_ring.dma.paddr;
454 XRWRITE4(sc, XHCI_ERDP_LO(0), (uint32_t)paddr)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0038 + (0x20 * (0))))), (((uint32_t)paddr))));
455 XRWRITE4(sc, XHCI_ERDP_HI(0), (uint32_t)(paddr >> 32))(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x003c + (0x20 * (0))))), (((uint32_t)(paddr >> 32
)))));
456
457 DPRINTF(("%s: ERDP=%#x%#x\n", DEVNAME(sc),
458 XRREAD4(sc, XHCI_ERDP_HI(0)), XRREAD4(sc, XHCI_ERDP_LO(0))));
459
460 /* Enable interrupts. */
461 hcr = XRREAD4(sc, XHCI_IMAN(0))(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0020 + (0x20 * (0)))))));
462 XRWRITE4(sc, XHCI_IMAN(0), hcr | XHCI_IMAN_INTR_ENA)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0020 + (0x20 * (0))))), ((hcr | 0x00000002))));
463
464 /* Set default interrupt moderation. */
465 XRWRITE4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0024 + (0x20 * (0))))), ((0x000001F4U))));
466
467 /* Allow event interrupt and start the controller. */
468 XOWRITE4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x00)), ((0x00000004|0x00000001))));
469
470 DPRINTF(("%s: USBCMD=%#x\n", DEVNAME(sc), XOREAD4(sc, XHCI_USBCMD)));
471 DPRINTF(("%s: IMAN=%#x\n", DEVNAME(sc), XRREAD4(sc, XHCI_IMAN(0))));
472}
473
474int
475xhci_detach(struct device *self, int flags)
476{
477 struct xhci_softc *sc = (struct xhci_softc *)self;
478 int rv;
479
480 rv = config_detach_children(self, flags);
481 if (rv != 0) {
482 printf("%s: error while detaching %d\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname), rv);
483 return (rv);
484 }
485
486 /* Since the hardware might already be gone, ignore the errors. */
487 xhci_command_abort(sc);
488
489 xhci_reset(sc);
490
491 /* Disable interrupts. */
492 XRWRITE4(sc, XHCI_IMOD(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0024 + (0x20 * (0))))), ((0))));
493 XRWRITE4(sc, XHCI_IMAN(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0020 + (0x20 * (0))))), ((0))));
494
495 /* Clear the event ring address. */
496 XRWRITE4(sc, XHCI_ERDP_LO(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0038 + (0x20 * (0))))), ((0))));
497 XRWRITE4(sc, XHCI_ERDP_HI(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x003c + (0x20 * (0))))), ((0))));
498
499 XRWRITE4(sc, XHCI_ERSTBA_LO(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0030 + (0x20 * (0))))), ((0))));
500 XRWRITE4(sc, XHCI_ERSTBA_HI(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0034 + (0x20 * (0))))), ((0))));
501
502 XRWRITE4(sc, XHCI_ERSTSZ(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0028 + (0x20 * (0))))), ((0))));
503
504 /* Clear the command ring address. */
505 XOWRITE4(sc, XHCI_CRCR_LO, 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x18)), ((0))));
506 XOWRITE4(sc, XHCI_CRCR_HI, 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x1C)), ((0))));
507
508 XOWRITE4(sc, XHCI_DCBAAP_LO, 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x30)), ((0))));
509 XOWRITE4(sc, XHCI_DCBAAP_HI, 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x34)), ((0))));
510
511 if (sc->sc_spad.npage > 0)
512 xhci_scratchpad_free(sc);
513
514 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_erst.dma);
515 xhci_ring_free(sc, &sc->sc_evt_ring);
516 xhci_ring_free(sc, &sc->sc_cmd_ring);
517 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_dcbaa.dma);
518
519 return (0);
520}
521
522int
523xhci_activate(struct device *self, int act)
524{
525 struct xhci_softc *sc = (struct xhci_softc *)self;
526 int rv = 0;
527
528 switch (act) {
529 case DVACT_RESUME4:
530 sc->sc_bus.use_polling++;
531 xhci_reinit(sc);
532 sc->sc_bus.use_polling--;
533 rv = config_activate_children(self, act);
534 break;
535 case DVACT_POWERDOWN6:
536 rv = config_activate_children(self, act);
537 xhci_reset(sc);
538 break;
539 default:
540 rv = config_activate_children(self, act);
541 break;
542 }
543
544 return (rv);
545}
546
547int
548xhci_reset(struct xhci_softc *sc)
549{
550 uint32_t hcr;
551 int i;
552
553 XOWRITE4(sc, XHCI_USBCMD, 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x00)), ((0)))); /* Halt controller */
554 for (i = 0; i < 100; i++) {
555 usb_delay_ms(&sc->sc_bus, 1);
556 hcr = XOREAD4(sc, XHCI_USBSTS)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x04)))) & XHCI_STS_HCH0x00000001;
557 if (hcr)
558 break;
559 }
560
561 if (!hcr)
562 printf("%s: halt timeout\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
563
564 XOWRITE4(sc, XHCI_USBCMD, XHCI_CMD_HCRST)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x00)), ((0x00000002))));
565 for (i = 0; i < 100; i++) {
566 usb_delay_ms(&sc->sc_bus, 1);
567 hcr = (XOREAD4(sc, XHCI_USBCMD)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x00)))) & XHCI_CMD_HCRST0x00000002) |
568 (XOREAD4(sc, XHCI_USBSTS)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x04)))) & XHCI_STS_CNR0x00000800);
569 if (!hcr)
570 break;
571 }
572
573 if (hcr) {
574 printf("%s: reset timeout\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
575 return (EIO5);
576 }
577
578 return (0);
579}
580
581void
582xhci_reinit(struct xhci_softc *sc)
583{
584 xhci_reset(sc);
585 xhci_ring_reset(sc, &sc->sc_cmd_ring);
586 xhci_ring_reset(sc, &sc->sc_evt_ring);
587
588 /* Renesas controllers, at least, need more time to resume. */
589 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT50);
590
591 xhci_config(sc);
592}
593
594int
595xhci_intr(void *v)
596{
597 struct xhci_softc *sc = v;
598
599 if (sc->sc_dead)
600 return (0);
601
602 /* If we get an interrupt while polling, then just ignore it. */
603 if (sc->sc_bus.use_polling) {
604 DPRINTFN(16, ("xhci_intr: ignored interrupt while polling\n"));
605 return (0);
606 }
607
608 return (xhci_intr1(sc));
609}
610
611int
612xhci_intr1(struct xhci_softc *sc)
613{
614 uint32_t intrs;
615
616 intrs = XOREAD4(sc, XHCI_USBSTS)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x04))));
617 if (intrs == 0xffffffff) {
618 sc->sc_bus.dying = 1;
619 sc->sc_dead = 1;
620 return (0);
621 }
622
623 if ((intrs & XHCI_STS_EINT0x00000008) == 0)
624 return (0);
625
626 sc->sc_bus.no_intrs++;
627
628 if (intrs & XHCI_STS_HSE0x00000004) {
629 printf("%s: host system error\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
630 sc->sc_bus.dying = 1;
631 XOWRITE4(sc, XHCI_USBSTS, intrs)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x04)), ((intrs))));
632 return (1);
633 }
634
635 /* Acknowledge interrupts */
636 XOWRITE4(sc, XHCI_USBSTS, intrs)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x04)), ((intrs))));
637 intrs = XRREAD4(sc, XHCI_IMAN(0))(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0020 + (0x20 * (0)))))));
638 XRWRITE4(sc, XHCI_IMAN(0), intrs | XHCI_IMAN_INTR_PEND)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0020 + (0x20 * (0))))), ((intrs | 0x00000001))));
639
640 usb_schedsoftintr(&sc->sc_bus);
641
642 return (1);
643}
644
645void
646xhci_poll(struct usbd_bus *bus)
647{
648 struct xhci_softc *sc = (struct xhci_softc *)bus;
649
650 if (XOREAD4(sc, XHCI_USBSTS)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x04)))))
651 xhci_intr1(sc);
652}
653
654void
655xhci_softintr(void *v)
656{
657 struct xhci_softc *sc = v;
658
659 if (sc->sc_bus.dying)
660 return;
661
662 sc->sc_bus.intr_context++;
663 xhci_event_dequeue(sc);
664 sc->sc_bus.intr_context--;
665}
666
667void
668xhci_event_dequeue(struct xhci_softc *sc)
669{
670 struct xhci_trb *trb;
671 uint64_t paddr;
672 uint32_t status, flags;
673
674 while ((trb = xhci_ring_consume(sc, &sc->sc_evt_ring)) != NULL((void *)0)) {
675 paddr = letoh64(trb->trb_paddr)((__uint64_t)(trb->trb_paddr));
676 status = letoh32(trb->trb_status)((__uint32_t)(trb->trb_status));
677 flags = letoh32(trb->trb_flags)((__uint32_t)(trb->trb_flags));
678
679 switch (flags & XHCI_TRB_TYPE_MASK0xfc00) {
680 case XHCI_EVT_XFER(32 << 10):
681 xhci_event_xfer(sc, paddr, status, flags);
682 break;
683 case XHCI_EVT_CMD_COMPLETE(33 << 10):
684 memcpy(&sc->sc_result_trb, trb, sizeof(*trb))__builtin_memcpy((&sc->sc_result_trb), (trb), (sizeof(
*trb)));
685 xhci_event_command(sc, paddr);
686 break;
687 case XHCI_EVT_PORT_CHANGE(34 << 10):
688 xhci_event_port_change(sc, paddr, status);
689 break;
690 case XHCI_EVT_HOST_CTRL(37 << 10):
691 /* TODO */
692 break;
693 default:
694#ifdef XHCI_DEBUG
695 printf("event (%d): ", XHCI_TRB_TYPE(flags)(((flags) & 0xfc00) >> 10));
696 xhci_dump_trb(trb);
697#endif
698 break;
699 }
700
701 }
702
703 paddr = (uint64_t)DEQPTR(sc->sc_evt_ring)((sc->sc_evt_ring).dma.paddr + (sizeof(struct xhci_trb) * (
sc->sc_evt_ring).index));
704 XRWRITE4(sc, XHCI_ERDP_LO(0), ((uint32_t)paddr) | XHCI_ERDP_LO_BUSY)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x0038 + (0x20 * (0))))), ((((uint32_t)paddr) | 0x00000008
))));
705 XRWRITE4(sc, XHCI_ERDP_HI(0), (uint32_t)(paddr >> 32))(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_runt_off
+ ((0x003c + (0x20 * (0))))), (((uint32_t)(paddr >> 32
)))));
706}
707
708void
709xhci_skip_all(struct xhci_pipe *xp)
710{
711 struct usbd_xfer *xfer, *last;
712
713 if (xp->skip) {
714 /*
715 * Find the last transfer to skip, this is necessary
716 * as xhci_xfer_done() posts new transfers which we
717 * don't want to skip
718 */
719 last = SIMPLEQ_FIRST(&xp->pipe.queue)((&xp->pipe.queue)->sqh_first);
720 if (last == NULL((void *)0))
721 goto done;
722 while ((xfer = SIMPLEQ_NEXT(last, next)((last)->next.sqe_next)) != NULL((void *)0))
723 last = xfer;
724
725 do {
726 xfer = SIMPLEQ_FIRST(&xp->pipe.queue)((&xp->pipe.queue)->sqh_first);
727 if (xfer == NULL((void *)0))
728 goto done;
729 DPRINTF(("%s: skipping %p\n", __func__, xfer));
730 xfer->status = USBD_NORMAL_COMPLETION;
731 xhci_xfer_done(xfer);
732 } while (xfer != last);
733 done:
734 xp->skip = 0;
735 }
736}
737
738void
739xhci_event_xfer(struct xhci_softc *sc, uint64_t paddr, uint32_t status,
740 uint32_t flags)
741{
742 struct xhci_pipe *xp;
743 struct usbd_xfer *xfer;
744 uint8_t dci, slot, code, xfertype;
745 uint32_t remain;
746 int trb_idx;
747
748 slot = XHCI_TRB_GET_SLOT(flags)(((flags) >> 24) & 0xff);
749 dci = XHCI_TRB_GET_EP(flags)(((flags) >> 16) & 0x1f);
750 if (slot > sc->sc_noslot) {
751 DPRINTF(("%s: incorrect slot (%u)\n", DEVNAME(sc), slot));
752 return;
753 }
754
755 xp = sc->sc_sdevs[slot].pipes[dci - 1];
756 if (xp == NULL((void *)0)) {
757 DPRINTF(("%s: incorrect dci (%u)\n", DEVNAME(sc), dci));
758 return;
759 }
760
761 code = XHCI_TRB_GET_CODE(status)(((status) >> 24) & 0xff);
762 remain = XHCI_TRB_REMAIN(status)((status) & 0xffffff);
763
764 switch (code) {
765 case XHCI_CODE_RING_UNDERRUN14:
766 DPRINTF(("%s: slot %u underrun with %zu TRB\n", DEVNAME(sc),
767 slot, xp->ring.ntrb - xp->free_trbs));
768 xhci_skip_all(xp);
769 return;
770 case XHCI_CODE_RING_OVERRUN15:
771 DPRINTF(("%s: slot %u overrun with %zu TRB\n", DEVNAME(sc),
772 slot, xp->ring.ntrb - xp->free_trbs));
773 xhci_skip_all(xp);
774 return;
775 case XHCI_CODE_MISSED_SRV23:
776 DPRINTF(("%s: slot %u missed srv with %zu TRB\n", DEVNAME(sc),
777 slot, xp->ring.ntrb - xp->free_trbs));
778 xp->skip = 1;
779 return;
780 default:
781 break;
782 }
783
784 trb_idx = (paddr - xp->ring.dma.paddr) / sizeof(struct xhci_trb);
785 if (trb_idx < 0 || trb_idx >= xp->ring.ntrb) {
786 printf("%s: wrong trb index (%u) max is %zu\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname),
787 trb_idx, xp->ring.ntrb - 1);
788 return;
789 }
790
791 xfer = xp->pending_xfers[trb_idx];
792 if (xfer == NULL((void *)0)) {
793 DPRINTF(("%s: NULL xfer pointer\n", DEVNAME(sc)));
794 return;
795 }
796
797 if (remain > xfer->length)
798 remain = xfer->length;
799
800 xfertype = UE_GET_XFERTYPE(xfer->pipe->endpoint->edesc->bmAttributes)((xfer->pipe->endpoint->edesc->bmAttributes) &
0x03);
801
802 switch (xfertype) {
803 case UE_BULK0x02:
804 case UE_INTERRUPT0x03:
805 case UE_CONTROL0x00:
806 if (xhci_event_xfer_generic(sc, xfer, xp, remain, trb_idx,
807 code, slot, dci))
808 return;
809 break;
810 case UE_ISOCHRONOUS0x01:
811 if (xhci_event_xfer_isoc(xfer, xp, remain, trb_idx, code))
812 return;
813 break;
814 default:
815 panic("xhci_event_xfer: unknown xfer type %u", xfertype);
816 }
817
818 xhci_xfer_done(xfer);
819}
820
821uint32_t
822xhci_xfer_length_generic(struct xhci_xfer *xx, struct xhci_pipe *xp,
823 int trb_idx)
824{
825 int trb0_idx;
826 uint32_t len = 0, type;
827
828 trb0_idx =
829 ((xx->index + xp->ring.ntrb) - xx->ntrb) % (xp->ring.ntrb - 1);
830
831 while (1) {
832 type = letoh32(xp->ring.trbs[trb0_idx].trb_flags)((__uint32_t)(xp->ring.trbs[trb0_idx].trb_flags)) &
833 XHCI_TRB_TYPE_MASK0xfc00;
834 if (type == XHCI_TRB_TYPE_NORMAL(1 << 10) || type == XHCI_TRB_TYPE_DATA(3 << 10))
835 len += XHCI_TRB_LEN(letoh32(((((__uint32_t)(xp->ring.trbs[trb0_idx].trb_status))) &
0x1ffff)
836 xp->ring.trbs[trb0_idx].trb_status))((((__uint32_t)(xp->ring.trbs[trb0_idx].trb_status))) &
0x1ffff);
837 if (trb0_idx == trb_idx)
838 break;
839 if (++trb0_idx == xp->ring.ntrb)
840 trb0_idx = 0;
841 }
842 return len;
843}
844
845int
846xhci_event_xfer_generic(struct xhci_softc *sc, struct usbd_xfer *xfer,
847 struct xhci_pipe *xp, uint32_t remain, int trb_idx,
848 uint8_t code, uint8_t slot, uint8_t dci)
849{
850 struct xhci_xfer *xx = (struct xhci_xfer *)xfer;
851
852 switch (code) {
853 case XHCI_CODE_SUCCESS1:
854 if (xfer->actlen == 0) {
855 if (remain)
856 xfer->actlen =
857 xhci_xfer_length_generic(xx, xp, trb_idx) -
858 remain;
859 else
860 xfer->actlen = xfer->length;
861 }
862 if (xfer->actlen)
863 usb_syncmem(&xfer->dmabuf, 0, xfer->actlen,
864 usbd_xfer_isread(xfer) ?
865 BUS_DMASYNC_POSTREAD0x02 : BUS_DMASYNC_POSTWRITE0x08);
866 xfer->status = USBD_NORMAL_COMPLETION;
867 break;
868 case XHCI_CODE_SHORT_XFER13:
869 /*
870 * Use values from the transfer TRB instead of the status TRB.
871 */
872 if (xfer->actlen == 0)
873 xfer->actlen =
874 xhci_xfer_length_generic(xx, xp, trb_idx) - remain;
875 /*
876 * If this is not the last TRB of a transfer, we should
877 * theoretically clear the IOC at the end of the chain
878 * but the HC might have already processed it before we
879 * had a chance to schedule the softinterrupt.
880 */
881 if (xx->index != trb_idx) {
882 DPRINTF(("%s: short xfer %p for %u\n",
883 DEVNAME(sc), xfer, xx->index));
884 return (1);
885 }
886 if (xfer->actlen)
887 usb_syncmem(&xfer->dmabuf, 0, xfer->actlen,
888 usbd_xfer_isread(xfer) ?
889 BUS_DMASYNC_POSTREAD0x02 : BUS_DMASYNC_POSTWRITE0x08);
890 xfer->status = USBD_NORMAL_COMPLETION;
891 break;
892 case XHCI_CODE_TXERR4:
893 case XHCI_CODE_SPLITERR36:
894 DPRINTF(("%s: txerr? code %d\n", DEVNAME(sc), code));
895 xfer->status = USBD_IOERROR;
896 break;
897 case XHCI_CODE_STALL6:
898 case XHCI_CODE_BABBLE3:
899 DPRINTF(("%s: babble code %d\n", DEVNAME(sc), code));
900 /* Prevent any timeout to kick in. */
901 timeout_del(&xfer->timeout_handle);
902 usb_rem_task(xfer->device, &xfer->abort_task);
903
904 /* We need to report this condition for umass(4). */
905 if (code == XHCI_CODE_STALL6)
906 xp->halted = USBD_STALLED;
907 else
908 xp->halted = USBD_IOERROR;
909 /*
910 * Since the stack might try to start a new transfer as
911 * soon as a pending one finishes, make sure the endpoint
912 * is fully reset before calling usb_transfer_complete().
913 */
914 xp->aborted_xfer = xfer;
915 xhci_cmd_reset_ep_async(sc, slot, dci);
916 return (1);
917 case XHCI_CODE_XFER_STOPPED26:
918 case XHCI_CODE_XFER_STOPINV27:
919 /* Endpoint stopped while processing a TD. */
920 if (xfer == xp->aborted_xfer) {
921 DPRINTF(("%s: stopped xfer=%p\n", __func__, xfer));
922 return (1);
923 }
924
925 /* FALLTHROUGH */
926 default:
927 DPRINTF(("%s: unhandled code %d\n", DEVNAME(sc), code));
928 xfer->status = USBD_IOERROR;
929 xp->halted = 1;
930 break;
931 }
932
933 return (0);
934}
935
936int
937xhci_event_xfer_isoc(struct usbd_xfer *xfer, struct xhci_pipe *xp,
938 uint32_t remain, int trb_idx, uint8_t code)
939{
940 struct usbd_xfer *skipxfer;
941 struct xhci_xfer *xx = (struct xhci_xfer *)xfer;
942 int trb0_idx, frame_idx = 0, skip_trb = 0;
943
944 KASSERT(xx->index >= 0)((xx->index >= 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c"
, 944, "xx->index >= 0"));
945
946 switch (code) {
947 case XHCI_CODE_SHORT_XFER13:
948 xp->trb_processed[trb_idx] = TRB_PROCESSED_SHORT2;
949 break;
950 default:
951 xp->trb_processed[trb_idx] = TRB_PROCESSED_YES1;
952 break;
953 }
954
955 trb0_idx =
956 ((xx->index + xp->ring.ntrb) - xx->ntrb) % (xp->ring.ntrb - 1);
957
958 /* Find the according frame index for this TRB. */
959 while (trb0_idx != trb_idx) {
960 if ((letoh32(xp->ring.trbs[trb0_idx].trb_flags)((__uint32_t)(xp->ring.trbs[trb0_idx].trb_flags)) &
961 XHCI_TRB_TYPE_MASK0xfc00) == XHCI_TRB_TYPE_ISOCH(5 << 10))
962 frame_idx++;
963 if (trb0_idx++ == (xp->ring.ntrb - 1))
964 trb0_idx = 0;
965 }
966
967 /*
968 * If we queued two TRBs for a frame and this is the second TRB,
969 * check if the first TRB needs accounting since it might not have
970 * raised an interrupt in case of full data received.
971 */
972 if ((letoh32(xp->ring.trbs[trb_idx].trb_flags)((__uint32_t)(xp->ring.trbs[trb_idx].trb_flags)) & XHCI_TRB_TYPE_MASK0xfc00) ==
973 XHCI_TRB_TYPE_NORMAL(1 << 10)) {
974 frame_idx--;
975 if (trb_idx == 0)
976 trb0_idx = xp->ring.ntrb - 2;
977 else
978 trb0_idx = trb_idx - 1;
979 if (xp->trb_processed[trb0_idx] == TRB_PROCESSED_NO0) {
980 xfer->frlengths[frame_idx] = XHCI_TRB_LEN(letoh32(((((__uint32_t)(xp->ring.trbs[trb0_idx].trb_status))) &
0x1ffff)
981 xp->ring.trbs[trb0_idx].trb_status))((((__uint32_t)(xp->ring.trbs[trb0_idx].trb_status))) &
0x1ffff);
982 } else if (xp->trb_processed[trb0_idx] == TRB_PROCESSED_SHORT2) {
983 skip_trb = 1;
984 }
985 }
986
987 if (!skip_trb) {
988 xfer->frlengths[frame_idx] +=
989 XHCI_TRB_LEN(letoh32(xp->ring.trbs[trb_idx].trb_status))((((__uint32_t)(xp->ring.trbs[trb_idx].trb_status))) &
0x1ffff) -
990 remain;
991 xfer->actlen += xfer->frlengths[frame_idx];
992 }
993
994 if (xx->index != trb_idx)
995 return (1);
996
997 if (xp->skip) {
998 while (1) {
999 skipxfer = SIMPLEQ_FIRST(&xp->pipe.queue)((&xp->pipe.queue)->sqh_first);
1000 if (skipxfer == xfer || skipxfer == NULL((void *)0))
1001 break;
1002 DPRINTF(("%s: skipping %p\n", __func__, skipxfer));
1003 skipxfer->status = USBD_NORMAL_COMPLETION;
1004 xhci_xfer_done(skipxfer);
1005 }
1006 xp->skip = 0;
1007 }
1008
1009 usb_syncmem(&xfer->dmabuf, 0, xfer->length,
1010 usbd_xfer_isread(xfer) ?
1011 BUS_DMASYNC_POSTREAD0x02 : BUS_DMASYNC_POSTWRITE0x08);
1012 xfer->status = USBD_NORMAL_COMPLETION;
1013
1014 return (0);
1015}
1016
1017void
1018xhci_event_command(struct xhci_softc *sc, uint64_t paddr)
1019{
1020 struct xhci_trb *trb;
1021 struct xhci_pipe *xp;
1022 uint32_t flags;
1023 uint8_t dci, slot;
1024 int trb_idx, status;
1025
1026 trb_idx = (paddr - sc->sc_cmd_ring.dma.paddr) / sizeof(*trb);
1027 if (trb_idx < 0 || trb_idx >= sc->sc_cmd_ring.ntrb) {
1028 printf("%s: wrong trb index (%u) max is %zu\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname),
1029 trb_idx, sc->sc_cmd_ring.ntrb - 1);
1030 return;
1031 }
1032
1033 trb = &sc->sc_cmd_ring.trbs[trb_idx];
1034
1035 bus_dmamap_sync(sc->sc_cmd_ring.dma.tag, sc->sc_cmd_ring.dma.map,(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x02 | 0x08))
1036 TRBOFF(&sc->sc_cmd_ring, trb), sizeof(struct xhci_trb),(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x02 | 0x08))
1037 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x02 | 0x08));
1038
1039 flags = letoh32(trb->trb_flags)((__uint32_t)(trb->trb_flags));
1040
1041 slot = XHCI_TRB_GET_SLOT(flags)(((flags) >> 24) & 0xff);
1042 dci = XHCI_TRB_GET_EP(flags)(((flags) >> 16) & 0x1f);
1043
1044 switch (flags & XHCI_TRB_TYPE_MASK0xfc00) {
1045 case XHCI_CMD_RESET_EP(14 << 10):
1046 xp = sc->sc_sdevs[slot].pipes[dci - 1];
1047 if (xp == NULL((void *)0))
1048 break;
1049
1050 /* Update the dequeue pointer past the last TRB. */
1051 xhci_cmd_set_tr_deq_async(sc, xp->slot, xp->dci,
1052 DEQPTR(xp->ring)((xp->ring).dma.paddr + (sizeof(struct xhci_trb) * (xp->
ring).index)) | xp->ring.toggle);
1053 break;
1054 case XHCI_CMD_SET_TR_DEQ(16 << 10):
1055 xp = sc->sc_sdevs[slot].pipes[dci - 1];
1056 if (xp == NULL((void *)0))
1057 break;
1058
1059 status = xp->halted;
1060 xp->halted = 0;
1061 if (xp->aborted_xfer != NULL((void *)0)) {
1062 xp->aborted_xfer->status = status;
1063 xhci_xfer_done(xp->aborted_xfer);
1064 wakeup(xp);
1065 }
1066 break;
1067 case XHCI_CMD_CONFIG_EP(12 << 10):
1068 case XHCI_CMD_STOP_EP(15 << 10):
1069 case XHCI_CMD_DISABLE_SLOT(10 << 10):
1070 case XHCI_CMD_ENABLE_SLOT(9 << 10):
1071 case XHCI_CMD_ADDRESS_DEVICE(11 << 10):
1072 case XHCI_CMD_EVAL_CTX(13 << 10):
1073 case XHCI_CMD_NOOP(23 << 10):
1074 /*
1075 * All these commands are synchronous.
1076 *
1077 * If TRBs differ, this could be a delayed result after we
1078 * gave up waiting for the expected TRB due to timeout.
1079 */
1080 if (sc->sc_cmd_trb == trb) {
1081 sc->sc_cmd_trb = NULL((void *)0);
1082 wakeup(&sc->sc_cmd_trb);
1083 }
1084 break;
1085 default:
1086 DPRINTF(("%s: unexpected command %x\n", DEVNAME(sc), flags));
1087 }
1088}
1089
1090void
1091xhci_event_port_change(struct xhci_softc *sc, uint64_t paddr, uint32_t status)
1092{
1093 struct usbd_xfer *xfer = sc->sc_intrxfer;
1094 uint32_t port = XHCI_TRB_PORTID(paddr)(((paddr) & (0xff << 24)) >> 24);
1095 uint8_t *p;
1096
1097 if (XHCI_TRB_GET_CODE(status)(((status) >> 24) & 0xff) != XHCI_CODE_SUCCESS1) {
1098 DPRINTF(("%s: failed port status event\n", DEVNAME(sc)));
1099 return;
1100 }
1101
1102 if (xfer == NULL((void *)0))
1103 return;
1104
1105 p = KERNADDR(&xfer->dmabuf, 0)((void *)((char *)((&xfer->dmabuf)->block->kaddr
+ (&xfer->dmabuf)->offs) + (0)));
1106 memset(p, 0, xfer->length)__builtin_memset((p), (0), (xfer->length));
1107
1108 p[port/8] |= 1 << (port%8);
1109 DPRINTF(("%s: port=%d change=0x%02x\n", DEVNAME(sc), port, *p));
1110
1111 xfer->actlen = xfer->length;
1112 xfer->status = USBD_NORMAL_COMPLETION;
1113
1114 usb_transfer_complete(xfer);
1115}
1116
1117void
1118xhci_xfer_done(struct usbd_xfer *xfer)
1119{
1120 struct xhci_pipe *xp = (struct xhci_pipe *)xfer->pipe;
1121 struct xhci_xfer *xx = (struct xhci_xfer *)xfer;
1122 int ntrb, i;
1123
1124 splsoftassert(IPL_SOFTUSB)do { if (splassert_ctl > 0) { splassert_check(0x2, __func__
); } } while (0);
1125
1126#ifdef XHCI_DEBUG
1127 if (xx->index < 0 || xp->pending_xfers[xx->index] == NULL((void *)0)) {
1128 printf("%s: xfer=%p done (idx=%d, ntrb=%zd)\n", __func__,
1129 xfer, xx->index, xx->ntrb);
1130 }
1131#endif
1132
1133 if (xp->aborted_xfer == xfer)
1134 xp->aborted_xfer = NULL((void *)0);
1135
1136 for (ntrb = 0, i = xx->index; ntrb < xx->ntrb; ntrb++, i--) {
1137 xp->pending_xfers[i] = NULL((void *)0);
1138 if (i == 0)
1139 i = (xp->ring.ntrb - 1);
1140 }
1141 xp->free_trbs += xx->ntrb;
1142 xp->free_trbs += xx->zerotd;
1143 xx->index = -1;
1144 xx->ntrb = 0;
1145 xx->zerotd = 0;
1146
1147 timeout_del(&xfer->timeout_handle);
1148 usb_rem_task(xfer->device, &xfer->abort_task);
1149 usb_transfer_complete(xfer);
1150}
1151
1152/*
1153 * Calculate the Device Context Index (DCI) for endpoints as stated
1154 * in section 4.5.1 of xHCI specification r1.1.
1155 */
1156static inline uint8_t
1157xhci_ed2dci(usb_endpoint_descriptor_t *ed)
1158{
1159 uint8_t dir;
1160
1161 if (UE_GET_XFERTYPE(ed->bmAttributes)((ed->bmAttributes) & 0x03) == UE_CONTROL0x00)
1162 return (UE_GET_ADDR(ed->bEndpointAddress)((ed->bEndpointAddress) & 0x0f) * 2 + 1);
1163
1164 if (UE_GET_DIR(ed->bEndpointAddress)((ed->bEndpointAddress) & 0x80) == UE_DIR_IN0x80)
1165 dir = 1;
1166 else
1167 dir = 0;
1168
1169 return (UE_GET_ADDR(ed->bEndpointAddress)((ed->bEndpointAddress) & 0x0f) * 2 + dir);
1170}
1171
1172usbd_status
1173xhci_pipe_open(struct usbd_pipe *pipe)
1174{
1175 struct xhci_softc *sc = (struct xhci_softc *)pipe->device->bus;
1176 struct xhci_pipe *xp = (struct xhci_pipe *)pipe;
1177 usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1178 uint8_t slot = 0, xfertype = UE_GET_XFERTYPE(ed->bmAttributes)((ed->bmAttributes) & 0x03);
1179 int error;
1180
1181 KASSERT(xp->slot == 0)((xp->slot == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c"
, 1181, "xp->slot == 0"));
1182
1183 if (sc->sc_bus.dying)
1184 return (USBD_IOERROR);
1185
1186 /* Root Hub */
1187 if (pipe->device->depth == 0) {
1188 switch (ed->bEndpointAddress) {
1189 case USB_CONTROL_ENDPOINT0:
1190 pipe->methods = &xhci_root_ctrl_methods;
1191 break;
1192 case UE_DIR_IN0x80 | XHCI_INTR_ENDPT1:
1193 pipe->methods = &xhci_root_intr_methods;
1194 break;
1195 default:
1196 pipe->methods = NULL((void *)0);
1197 return (USBD_INVAL);
1198 }
1199 return (USBD_NORMAL_COMPLETION);
1200 }
1201
1202#if 0
1203 /* Issue a noop to check if the command ring is correctly configured. */
1204 xhci_cmd_noop(sc);
1205#endif
1206
1207 switch (xfertype) {
1208 case UE_CONTROL0x00:
1209 pipe->methods = &xhci_device_ctrl_methods;
1210
1211 /*
1212 * Get a slot and init the device's contexts.
1213 *
1214 * Since the control endpoint, represented as the default
1215 * pipe, is always opened first we are dealing with a
1216 * new device. Put a new slot in the ENABLED state.
1217 *
1218 */
1219 error = xhci_cmd_slot_control(sc, &slot, 1);
1220 if (error || slot == 0 || slot > sc->sc_noslot)
1221 return (USBD_INVAL);
1222
1223 if (xhci_softdev_alloc(sc, slot)) {
1224 xhci_cmd_slot_control(sc, &slot, 0);
1225 return (USBD_NOMEM);
1226 }
1227
1228 break;
1229 case UE_ISOCHRONOUS0x01:
1230 pipe->methods = &xhci_device_isoc_methods;
1231 break;
1232 case UE_BULK0x02:
1233 pipe->methods = &xhci_device_bulk_methods;
1234 break;
1235 case UE_INTERRUPT0x03:
1236 pipe->methods = &xhci_device_intr_methods;
1237 break;
1238 default:
1239 return (USBD_INVAL);
1240 }
1241
1242 /*
1243 * Our USBD Bus Interface is pipe-oriented but for most of the
1244 * operations we need to access a device context, so keep track
1245 * of the slot ID in every pipe.
1246 */
1247 if (slot == 0)
1248 slot = ((struct xhci_pipe *)pipe->device->default_pipe)->slot;
1249
1250 xp->slot = slot;
1251 xp->dci = xhci_ed2dci(ed);
1252
1253 if (xhci_pipe_init(sc, pipe)) {
1254 xhci_cmd_slot_control(sc, &slot, 0);
1255 return (USBD_IOERROR);
1256 }
1257
1258 return (USBD_NORMAL_COMPLETION);
1259}
1260
1261/*
1262 * Set the maximum Endpoint Service Interface Time (ESIT) payload and
1263 * the average TRB buffer length for an endpoint.
1264 */
1265static inline uint32_t
1266xhci_get_txinfo(struct xhci_softc *sc, struct usbd_pipe *pipe)
1267{
1268 usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1269 uint32_t mep, atl, mps = UGETW(ed->wMaxPacketSize)(*(u_int16_t *)(ed->wMaxPacketSize));
1270
1271 switch (UE_GET_XFERTYPE(ed->bmAttributes)((ed->bmAttributes) & 0x03)) {
1272 case UE_CONTROL0x00:
1273 mep = 0;
1274 atl = 8;
1275 break;
1276 case UE_INTERRUPT0x03:
1277 case UE_ISOCHRONOUS0x01:
1278 if (pipe->device->speed == USB_SPEED_SUPER4) {
1279 /* XXX Read the companion descriptor */
1280 }
1281
1282 mep = (UE_GET_TRANS(mps)(((mps) >> 11) & 0x3) + 1) * UE_GET_SIZE(mps)((mps) & 0x7ff);
1283 atl = mep;
1284 break;
1285 case UE_BULK0x02:
1286 default:
1287 mep = 0;
1288 atl = 0;
1289 }
1290
1291 return (XHCI_EPCTX_MAX_ESIT_PAYLOAD(mep)(((mep) & 0xffff) << 16) | XHCI_EPCTX_AVG_TRB_LEN(atl)((atl) & 0xffff));
1292}
1293
1294static inline uint32_t
1295xhci_linear_interval(usb_endpoint_descriptor_t *ed)
1296{
1297 uint32_t ival = min(max(1, ed->bInterval), 255);
1298
1299 return (fls(ival) - 1);
1300}
1301
1302static inline uint32_t
1303xhci_exponential_interval(usb_endpoint_descriptor_t *ed)
1304{
1305 uint32_t ival = min(max(1, ed->bInterval), 16);
1306
1307 return (ival - 1);
1308}
1309/*
1310 * Return interval for endpoint expressed in 2^(ival) * 125us.
1311 *
1312 * See section 6.2.3.6 of xHCI r1.1 Specification for more details.
1313 */
1314uint32_t
1315xhci_pipe_interval(struct usbd_pipe *pipe)
1316{
1317 usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1318 uint8_t speed = pipe->device->speed;
1319 uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes)((ed->bmAttributes) & 0x03);
1320 uint32_t ival;
1321
1322 if (xfertype == UE_CONTROL0x00 || xfertype == UE_BULK0x02) {
1323 /* Control and Bulk endpoints never NAKs. */
1324 ival = 0;
1325 } else {
1326 switch (speed) {
1327 case USB_SPEED_FULL2:
1328 if (xfertype == UE_ISOCHRONOUS0x01) {
1329 /* Convert 1-2^(15)ms into 3-18 */
1330 ival = xhci_exponential_interval(ed) + 3;
1331 break;
1332 }
1333 /* FALLTHROUGH */
1334 case USB_SPEED_LOW1:
1335 /* Convert 1-255ms into 3-10 */
1336 ival = xhci_linear_interval(ed) + 3;
1337 break;
1338 case USB_SPEED_HIGH3:
1339 case USB_SPEED_SUPER4:
1340 default:
1341 /* Convert 1-2^(15) * 125us into 0-15 */
1342 ival = xhci_exponential_interval(ed);
1343 break;
1344 }
1345 }
1346
1347 KASSERT(ival <= 15)((ival <= 15) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c"
, 1347, "ival <= 15"));
1348 return (XHCI_EPCTX_SET_IVAL(ival)(((ival) & 0xff) << 16));
1349}
1350
1351uint32_t
1352xhci_pipe_maxburst(struct usbd_pipe *pipe)
1353{
1354 usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1355 uint32_t mps = UGETW(ed->wMaxPacketSize)(*(u_int16_t *)(ed->wMaxPacketSize));
1356 uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes)((ed->bmAttributes) & 0x03);
1357 uint32_t maxb = 0;
1358
1359 switch (pipe->device->speed) {
1360 case USB_SPEED_HIGH3:
1361 if (xfertype == UE_ISOCHRONOUS0x01 || xfertype == UE_INTERRUPT0x03)
1362 maxb = UE_GET_TRANS(mps)(((mps) >> 11) & 0x3);
1363 break;
1364 case USB_SPEED_SUPER4:
1365 /* XXX Read the companion descriptor */
1366 default:
1367 break;
1368 }
1369
1370 return (maxb);
1371}
1372
1373static inline uint32_t
1374xhci_last_valid_dci(struct xhci_pipe **pipes, struct xhci_pipe *ignore)
1375{
1376 struct xhci_pipe *lxp;
1377 int i;
1378
1379 /* Find the last valid Endpoint Context. */
1380 for (i = 30; i >= 0; i--) {
1381 lxp = pipes[i];
1382 if (lxp != NULL((void *)0) && lxp != ignore)
1383 return XHCI_SCTX_DCI(lxp->dci)(((lxp->dci) & 0x1f) << 27);
1384 }
1385
1386 return 0;
1387}
1388
1389int
1390xhci_context_setup(struct xhci_softc *sc, struct usbd_pipe *pipe)
1391{
1392 struct xhci_pipe *xp = (struct xhci_pipe *)pipe;
1393 struct xhci_soft_dev *sdev = &sc->sc_sdevs[xp->slot];
1394 usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1395 uint32_t mps = UGETW(ed->wMaxPacketSize)(*(u_int16_t *)(ed->wMaxPacketSize));
1396 uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes)((ed->bmAttributes) & 0x03);
1397 uint8_t speed, cerr = 0;
1398 uint32_t route = 0, rhport = 0;
1399 struct usbd_device *hub;
1400
1401 /*
1402 * Calculate the Route String. Assume that there is no hub with
1403 * more than 15 ports and that they all have a detph < 6. See
1404 * section 8.9 of USB 3.1 Specification for more details.
1405 */
1406 for (hub = pipe->device; hub->myhub->depth; hub = hub->myhub) {
1407 uint32_t port = hub->powersrc->portno;
1408 uint32_t depth = hub->myhub->depth;
1409
1410 route |= port << (4 * (depth - 1));
1411 }
1412
1413 /* Get Root Hub port */
1414 rhport = hub->powersrc->portno;
1415
1416 switch (pipe->device->speed) {
1417 case USB_SPEED_LOW1:
1418 speed = XHCI_SPEED_LOW2;
1419 break;
1420 case USB_SPEED_FULL2:
1421 speed = XHCI_SPEED_FULL1;
1422 break;
1423 case USB_SPEED_HIGH3:
1424 speed = XHCI_SPEED_HIGH3;
1425 break;
1426 case USB_SPEED_SUPER4:
1427 speed = XHCI_SPEED_SUPER4;
1428 break;
1429 default:
1430 return (USBD_INVAL);
1431 }
1432
1433 /* Setup the endpoint context */
1434 if (xfertype != UE_ISOCHRONOUS0x01)
1435 cerr = 3;
1436
1437 if ((ed->bEndpointAddress & UE_DIR_IN0x80) || (xfertype == UE_CONTROL0x00))
1438 xfertype |= 0x4;
1439
1440 sdev->ep_ctx[xp->dci-1]->info_lo = htole32(xhci_pipe_interval(pipe))((__uint32_t)(xhci_pipe_interval(pipe)));
1441 sdev->ep_ctx[xp->dci-1]->info_hi = htole32(((__uint32_t)((((((mps) & 0x7ff)) & 0xffff) << 16
) | (((xhci_pipe_maxburst(pipe)) & 0xff) << 8) | ((
(xfertype) & 0x7) << 3) | (((cerr) & 0x3) <<
1)))
1442 XHCI_EPCTX_SET_MPS(UE_GET_SIZE(mps)) |((__uint32_t)((((((mps) & 0x7ff)) & 0xffff) << 16
) | (((xhci_pipe_maxburst(pipe)) & 0xff) << 8) | ((
(xfertype) & 0x7) << 3) | (((cerr) & 0x3) <<
1)))
1443 XHCI_EPCTX_SET_MAXB(xhci_pipe_maxburst(pipe)) |((__uint32_t)((((((mps) & 0x7ff)) & 0xffff) << 16
) | (((xhci_pipe_maxburst(pipe)) & 0xff) << 8) | ((
(xfertype) & 0x7) << 3) | (((cerr) & 0x3) <<
1)))
1444 XHCI_EPCTX_SET_EPTYPE(xfertype) | XHCI_EPCTX_SET_CERR(cerr)((__uint32_t)((((((mps) & 0x7ff)) & 0xffff) << 16
) | (((xhci_pipe_maxburst(pipe)) & 0xff) << 8) | ((
(xfertype) & 0x7) << 3) | (((cerr) & 0x3) <<
1)))
1445 )((__uint32_t)((((((mps) & 0x7ff)) & 0xffff) << 16
) | (((xhci_pipe_maxburst(pipe)) & 0xff) << 8) | ((
(xfertype) & 0x7) << 3) | (((cerr) & 0x3) <<
1)));
1446 sdev->ep_ctx[xp->dci-1]->txinfo = htole32(xhci_get_txinfo(sc, pipe))((__uint32_t)(xhci_get_txinfo(sc, pipe)));
1447 sdev->ep_ctx[xp->dci-1]->deqp = htole64(((__uint64_t)(((xp->ring).dma.paddr + (sizeof(struct xhci_trb
) * (xp->ring).index)) | xp->ring.toggle))
1448 DEQPTR(xp->ring) | xp->ring.toggle((__uint64_t)(((xp->ring).dma.paddr + (sizeof(struct xhci_trb
) * (xp->ring).index)) | xp->ring.toggle))
1449 )((__uint64_t)(((xp->ring).dma.paddr + (sizeof(struct xhci_trb
) * (xp->ring).index)) | xp->ring.toggle));
1450
1451 /* Unmask the new endpoint */
1452 sdev->input_ctx->drop_flags = 0;
1453 sdev->input_ctx->add_flags = htole32(XHCI_INCTX_MASK_DCI(xp->dci))((__uint32_t)((0x1 << (xp->dci))));
1454
1455 /* Setup the slot context */
1456 sdev->slot_ctx->info_lo = htole32(((__uint32_t)(xhci_last_valid_dci(sdev->pipes, ((void *)0)
) | (((speed) & 0xf) << 20) | ((route) & 0xfffff
)))
1457 xhci_last_valid_dci(sdev->pipes, NULL) | XHCI_SCTX_SPEED(speed) |((__uint32_t)(xhci_last_valid_dci(sdev->pipes, ((void *)0)
) | (((speed) & 0xf) << 20) | ((route) & 0xfffff
)))
1458 XHCI_SCTX_ROUTE(route)((__uint32_t)(xhci_last_valid_dci(sdev->pipes, ((void *)0)
) | (((speed) & 0xf) << 20) | ((route) & 0xfffff
)))
1459 )((__uint32_t)(xhci_last_valid_dci(sdev->pipes, ((void *)0)
) | (((speed) & 0xf) << 20) | ((route) & 0xfffff
)));
1460 sdev->slot_ctx->info_hi = htole32(XHCI_SCTX_RHPORT(rhport))((__uint32_t)((((rhport) & 0xff) << 16)));
1461 sdev->slot_ctx->tt = 0;
1462 sdev->slot_ctx->state = 0;
1463
1464/* XXX */
1465#define UHUB_IS_MTT(dev) (dev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT0x02)
1466 /*
1467 * If we are opening the interrupt pipe of a hub, update its
1468 * context before putting it in the CONFIGURED state.
1469 */
1470 if (pipe->device->hub != NULL((void *)0)) {
1471 int nports = pipe->device->hub->nports;
1472
1473 sdev->slot_ctx->info_lo |= htole32(XHCI_SCTX_HUB(1))((__uint32_t)((((1) & 0x1) << 26)));
1474 sdev->slot_ctx->info_hi |= htole32(XHCI_SCTX_NPORTS(nports))((__uint32_t)((((nports) & 0xff) << 24)));
1475
1476 if (UHUB_IS_MTT(pipe->device))
1477 sdev->slot_ctx->info_lo |= htole32(XHCI_SCTX_MTT(1))((__uint32_t)((((1) & 0x1) << 25)));
1478
1479 sdev->slot_ctx->tt |= htole32(((__uint32_t)((((pipe->device->hub->ttthink) & 0x3
) << 16)))
1480 XHCI_SCTX_TT_THINK_TIME(pipe->device->hub->ttthink)((__uint32_t)((((pipe->device->hub->ttthink) & 0x3
) << 16)))
1481 )((__uint32_t)((((pipe->device->hub->ttthink) & 0x3
) << 16)));
1482 }
1483
1484 /*
1485 * If this is a Low or Full Speed device below an external High
1486 * Speed hub, it needs some TT love.
1487 */
1488 if (speed < XHCI_SPEED_HIGH3 && pipe->device->myhsport != NULL((void *)0)) {
1489 struct usbd_device *hshub = pipe->device->myhsport->parent;
1490 uint8_t slot = ((struct xhci_pipe *)hshub->default_pipe)->slot;
1491
1492 if (UHUB_IS_MTT(hshub))
1493 sdev->slot_ctx->info_lo |= htole32(XHCI_SCTX_MTT(1))((__uint32_t)((((1) & 0x1) << 25)));
1494
1495 sdev->slot_ctx->tt |= htole32(((__uint32_t)(((slot) & 0xff) | (((pipe->device->myhsport
->portno) & 0xff) << 8)))
1496 XHCI_SCTX_TT_HUB_SID(slot) |((__uint32_t)(((slot) & 0xff) | (((pipe->device->myhsport
->portno) & 0xff) << 8)))
1497 XHCI_SCTX_TT_PORT_NUM(pipe->device->myhsport->portno)((__uint32_t)(((slot) & 0xff) | (((pipe->device->myhsport
->portno) & 0xff) << 8)))
1498 )((__uint32_t)(((slot) & 0xff) | (((pipe->device->myhsport
->portno) & 0xff) << 8)));
1499 }
1500#undef UHUB_IS_MTT
1501
1502 /* Unmask the slot context */
1503 sdev->input_ctx->add_flags |= htole32(XHCI_INCTX_MASK_DCI(0))((__uint32_t)((0x1 << (0))));
1504
1505 bus_dmamap_sync(sdev->ictx_dma.tag, sdev->ictx_dma.map, 0,(*(sdev->ictx_dma.tag)->_dmamap_sync)((sdev->ictx_dma
.tag), (sdev->ictx_dma.map), (0), (sc->sc_pagesize), (0x01
| 0x04))
1506 sc->sc_pagesize, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(sdev->ictx_dma.tag)->_dmamap_sync)((sdev->ictx_dma
.tag), (sdev->ictx_dma.map), (0), (sc->sc_pagesize), (0x01
| 0x04));
1507
1508 return (0);
1509}
1510
1511int
1512xhci_pipe_init(struct xhci_softc *sc, struct usbd_pipe *pipe)
1513{
1514 struct xhci_pipe *xp = (struct xhci_pipe *)pipe;
1515 struct xhci_soft_dev *sdev = &sc->sc_sdevs[xp->slot];
1516 int error;
1517
1518#ifdef XHCI_DEBUG
1519 struct usbd_device *dev = pipe->device;
1520 printf("%s: pipe=%p addr=%d depth=%d port=%d speed=%d dev %d dci %u"
1521 " (epAddr=0x%x)\n", __func__, pipe, dev->address, dev->depth,
1522 dev->powersrc->portno, dev->speed, xp->slot, xp->dci,
1523 pipe->endpoint->edesc->bEndpointAddress);
1524#endif
1525
1526 if (xhci_ring_alloc(sc, &xp->ring, XHCI_MAX_XFER(16 * 16), XHCI_XFER_RING_ALIGN16))
1527 return (ENOMEM12);
1528
1529 xp->free_trbs = xp->ring.ntrb;
1530 xp->halted = 0;
1531
1532 sdev->pipes[xp->dci - 1] = xp;
1533
1534 error = xhci_context_setup(sc, pipe);
1535 if (error)
1536 return (error);
1537
1538 if (xp->dci == 1) {
1539 /*
1540 * If we are opening the default pipe, the Slot should
1541 * be in the ENABLED state. Issue an "Address Device"
1542 * with BSR=1 to put the device in the DEFAULT state.
1543 * We cannot jump directly to the ADDRESSED state with
1544 * BSR=0 because some Low/Full speed devices won't accept
1545 * a SET_ADDRESS command before we've read their device
1546 * descriptor.
1547 */
1548 error = xhci_cmd_set_address(sc, xp->slot,
1549 sdev->ictx_dma.paddr, XHCI_TRB_BSR(1 << 9));
1550 } else {
1551 error = xhci_cmd_configure_ep(sc, xp->slot,
1552 sdev->ictx_dma.paddr);
1553 }
1554
1555 if (error) {
1556 xhci_ring_free(sc, &xp->ring);
1557 return (EIO5);
1558 }
1559
1560 return (0);
1561}
1562
1563void
1564xhci_pipe_close(struct usbd_pipe *pipe)
1565{
1566 struct xhci_softc *sc = (struct xhci_softc *)pipe->device->bus;
1567 struct xhci_pipe *xp = (struct xhci_pipe *)pipe;
1568 struct xhci_soft_dev *sdev = &sc->sc_sdevs[xp->slot];
1569
1570 /* Root Hub */
1571 if (pipe->device->depth == 0)
1572 return;
1573
1574 /* Mask the endpoint */
1575 sdev->input_ctx->drop_flags = htole32(XHCI_INCTX_MASK_DCI(xp->dci))((__uint32_t)((0x1 << (xp->dci))));
1576 sdev->input_ctx->add_flags = 0;
1577
1578 /* Update last valid Endpoint Context */
1579 sdev->slot_ctx->info_lo &= htole32(~XHCI_SCTX_DCI(31))((__uint32_t)(~(((31) & 0x1f) << 27)));
1580 sdev->slot_ctx->info_lo |= htole32(xhci_last_valid_dci(sdev->pipes, xp))((__uint32_t)(xhci_last_valid_dci(sdev->pipes, xp)));
1581
1582 /* Clear the Endpoint Context */
1583 memset(sdev->ep_ctx[xp->dci - 1], 0, sizeof(struct xhci_epctx))__builtin_memset((sdev->ep_ctx[xp->dci - 1]), (0), (sizeof
(struct xhci_epctx)));
1584
1585 bus_dmamap_sync(sdev->ictx_dma.tag, sdev->ictx_dma.map, 0,(*(sdev->ictx_dma.tag)->_dmamap_sync)((sdev->ictx_dma
.tag), (sdev->ictx_dma.map), (0), (sc->sc_pagesize), (0x01
| 0x04))
1586 sc->sc_pagesize, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(sdev->ictx_dma.tag)->_dmamap_sync)((sdev->ictx_dma
.tag), (sdev->ictx_dma.map), (0), (sc->sc_pagesize), (0x01
| 0x04));
1587
1588 if (xhci_cmd_configure_ep(sc, xp->slot, sdev->ictx_dma.paddr))
1589 DPRINTF(("%s: error clearing ep (%d)\n", DEVNAME(sc), xp->dci));
1590
1591 xhci_ring_free(sc, &xp->ring);
1592 sdev->pipes[xp->dci - 1] = NULL((void *)0);
1593
1594 /*
1595 * If we are closing the default pipe, the device is probably
1596 * gone, so put its slot in the DISABLED state.
1597 */
1598 if (xp->dci == 1) {
1599 xhci_cmd_slot_control(sc, &xp->slot, 0);
1600 xhci_softdev_free(sc, xp->slot);
1601 }
1602}
1603
1604/*
1605 * Transition a device from DEFAULT to ADDRESSED Slot state, this hook
1606 * is needed for Low/Full speed devices.
1607 *
1608 * See section 4.5.3 of USB 3.1 Specification for more details.
1609 */
1610int
1611xhci_setaddr(struct usbd_device *dev, int addr)
1612{
1613 struct xhci_softc *sc = (struct xhci_softc *)dev->bus;
1614 struct xhci_pipe *xp = (struct xhci_pipe *)dev->default_pipe;
1615 struct xhci_soft_dev *sdev = &sc->sc_sdevs[xp->slot];
1616 int error;
1617
1618 /* Root Hub */
1619 if (dev->depth == 0)
1620 return (0);
1621
1622 KASSERT(xp->dci == 1)((xp->dci == 1) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c"
, 1622, "xp->dci == 1"));
1623
1624 error = xhci_context_setup(sc, dev->default_pipe);
1625 if (error)
1626 return (error);
1627
1628 error = xhci_cmd_set_address(sc, xp->slot, sdev->ictx_dma.paddr, 0);
1629
1630#ifdef XHCI_DEBUG
1631 if (error == 0) {
1632 struct xhci_sctx *sctx;
1633 uint8_t addr;
1634
1635 bus_dmamap_sync(sdev->octx_dma.tag, sdev->octx_dma.map, 0,(*(sdev->octx_dma.tag)->_dmamap_sync)((sdev->octx_dma
.tag), (sdev->octx_dma.map), (0), (sc->sc_pagesize), (0x02
))
1636 sc->sc_pagesize, BUS_DMASYNC_POSTREAD)(*(sdev->octx_dma.tag)->_dmamap_sync)((sdev->octx_dma
.tag), (sdev->octx_dma.map), (0), (sc->sc_pagesize), (0x02
));
1637
1638 /* Get output slot context. */
1639 sctx = (struct xhci_sctx *)sdev->octx_dma.vaddr;
1640 addr = XHCI_SCTX_DEV_ADDR(letoh32(sctx->state))((((__uint32_t)(sctx->state))) & 0xff);
1641 error = (addr == 0);
1642
1643 printf("%s: dev %d addr %d\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname), xp->slot, addr);
1644 }
1645#endif
1646
1647 return (error);
1648}
1649
1650struct usbd_xfer *
1651xhci_allocx(struct usbd_bus *bus)
1652{
1653 return (pool_get(xhcixfer, PR_NOWAIT0x0002 | PR_ZERO0x0008));
1654}
1655
1656void
1657xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer)
1658{
1659 pool_put(xhcixfer, xfer);
1660}
1661
1662int
1663xhci_scratchpad_alloc(struct xhci_softc *sc, int npage)
1664{
1665 uint64_t *pte;
1666 int error, i;
1667
1668 /* Allocate the required entry for the table. */
1669 error = usbd_dma_contig_alloc(&sc->sc_bus, &sc->sc_spad.table_dma,
1670 (void **)&pte, npage * sizeof(uint64_t), XHCI_SPAD_TABLE_ALIGN64,
1671 sc->sc_pagesize);
1672 if (error)
1673 return (ENOMEM12);
1674
1675 /* Allocate pages. XXX does not need to be contiguous. */
1676 error = usbd_dma_contig_alloc(&sc->sc_bus, &sc->sc_spad.pages_dma,
1677 NULL((void *)0), npage * sc->sc_pagesize, sc->sc_pagesize, 0);
1678 if (error) {
1679 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_spad.table_dma);
1680 return (ENOMEM12);
1681 }
1682
1683 for (i = 0; i < npage; i++) {
1684 pte[i] = htole64(((__uint64_t)(sc->sc_spad.pages_dma.paddr + (i * sc->sc_pagesize
)))
1685 sc->sc_spad.pages_dma.paddr + (i * sc->sc_pagesize)((__uint64_t)(sc->sc_spad.pages_dma.paddr + (i * sc->sc_pagesize
)))
1686 )((__uint64_t)(sc->sc_spad.pages_dma.paddr + (i * sc->sc_pagesize
)));
1687 }
1688
1689 bus_dmamap_sync(sc->sc_spad.table_dma.tag, sc->sc_spad.table_dma.map, 0,(*(sc->sc_spad.table_dma.tag)->_dmamap_sync)((sc->sc_spad
.table_dma.tag), (sc->sc_spad.table_dma.map), (0), (npage *
sizeof(uint64_t)), (0x01 | 0x04))
1690 npage * sizeof(uint64_t), BUS_DMASYNC_PREREAD |(*(sc->sc_spad.table_dma.tag)->_dmamap_sync)((sc->sc_spad
.table_dma.tag), (sc->sc_spad.table_dma.map), (0), (npage *
sizeof(uint64_t)), (0x01 | 0x04))
1691 BUS_DMASYNC_PREWRITE)(*(sc->sc_spad.table_dma.tag)->_dmamap_sync)((sc->sc_spad
.table_dma.tag), (sc->sc_spad.table_dma.map), (0), (npage *
sizeof(uint64_t)), (0x01 | 0x04));
1692
1693 /* Entry 0 points to the table of scratchpad pointers. */
1694 sc->sc_dcbaa.segs[0] = htole64(sc->sc_spad.table_dma.paddr)((__uint64_t)(sc->sc_spad.table_dma.paddr));
1695 bus_dmamap_sync(sc->sc_dcbaa.dma.tag, sc->sc_dcbaa.dma.map, 0,(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (0), (sizeof(uint64_t))
, (0x01 | 0x04))
1696 sizeof(uint64_t), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (0), (sizeof(uint64_t))
, (0x01 | 0x04));
1697
1698 sc->sc_spad.npage = npage;
1699
1700 return (0);
1701}
1702
1703void
1704xhci_scratchpad_free(struct xhci_softc *sc)
1705{
1706 sc->sc_dcbaa.segs[0] = 0;
1707 bus_dmamap_sync(sc->sc_dcbaa.dma.tag, sc->sc_dcbaa.dma.map, 0,(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (0), (sizeof(uint64_t))
, (0x01 | 0x04))
1708 sizeof(uint64_t), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (0), (sizeof(uint64_t))
, (0x01 | 0x04));
1709
1710 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_spad.pages_dma);
1711 usbd_dma_contig_free(&sc->sc_bus, &sc->sc_spad.table_dma);
1712}
1713
1714int
1715xhci_ring_alloc(struct xhci_softc *sc, struct xhci_ring *ring, size_t ntrb,
1716 size_t alignment)
1717{
1718 size_t size;
1719 int error;
1720
1721 size = ntrb * sizeof(struct xhci_trb);
1722
1723 error = usbd_dma_contig_alloc(&sc->sc_bus, &ring->dma,
1724 (void **)&ring->trbs, size, alignment, XHCI_RING_BOUNDARY(64 * 1024));
1725 if (error)
1726 return (error);
1727
1728 ring->ntrb = ntrb;
1729
1730 xhci_ring_reset(sc, ring);
1731
1732 return (0);
1733}
1734
1735void
1736xhci_ring_free(struct xhci_softc *sc, struct xhci_ring *ring)
1737{
1738 usbd_dma_contig_free(&sc->sc_bus, &ring->dma);
1739}
1740
1741void
1742xhci_ring_reset(struct xhci_softc *sc, struct xhci_ring *ring)
1743{
1744 size_t size;
1745
1746 size = ring->ntrb * sizeof(struct xhci_trb);
1747
1748 memset(ring->trbs, 0, size)__builtin_memset((ring->trbs), (0), (size));
1749
1750 ring->index = 0;
1751 ring->toggle = XHCI_TRB_CYCLE(1 << 0);
1752
1753 /*
1754 * Since all our rings use only one segment, at least for
1755 * the moment, link their tail to their head.
1756 */
1757 if (ring != &sc->sc_evt_ring) {
1758 struct xhci_trb *trb = &ring->trbs[ring->ntrb - 1];
1759
1760 trb->trb_paddr = htole64(ring->dma.paddr)((__uint64_t)(ring->dma.paddr));
1761 trb->trb_flags = htole32(XHCI_TRB_TYPE_LINK | XHCI_TRB_LINKSEG |((__uint32_t)((6 << 10) | (1 << 1) | (1 << 0
)))
1762 XHCI_TRB_CYCLE)((__uint32_t)((6 << 10) | (1 << 1) | (1 << 0
)));
1763 bus_dmamap_sync(ring->dma.tag, ring->dma.map, 0, size,(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (0), (size), (0x04))
1764 BUS_DMASYNC_PREWRITE)(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (0), (size), (0x04));
1765 } else
1766 bus_dmamap_sync(ring->dma.tag, ring->dma.map, 0, size,(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (0), (size), (0x01 | 0x04))
1767 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (0), (size), (0x01 | 0x04));
1768}
1769
1770struct xhci_trb*
1771xhci_ring_consume(struct xhci_softc *sc, struct xhci_ring *ring)
1772{
1773 struct xhci_trb *trb = &ring->trbs[ring->index];
1774
1775 KASSERT(ring->index < ring->ntrb)((ring->index < ring->ntrb) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/usb/xhci.c", 1775, "ring->index < ring->ntrb"
));
1776
1777 bus_dmamap_sync(ring->dma.tag, ring->dma.map, TRBOFF(ring, trb),(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(trb) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02))
1778 sizeof(struct xhci_trb), BUS_DMASYNC_POSTREAD)(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(trb) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02));
1779
1780 /* Make sure this TRB can be consumed. */
1781 if (ring->toggle != (letoh32(trb->trb_flags)((__uint32_t)(trb->trb_flags)) & XHCI_TRB_CYCLE(1 << 0)))
1782 return (NULL((void *)0));
1783
1784 ring->index++;
1785
1786 if (ring->index == ring->ntrb) {
1787 ring->index = 0;
1788 ring->toggle ^= 1;
1789 }
1790
1791 return (trb);
1792}
1793
1794struct xhci_trb*
1795xhci_ring_produce(struct xhci_softc *sc, struct xhci_ring *ring)
1796{
1797 struct xhci_trb *lnk, *trb;
1798
1799 KASSERT(ring->index < ring->ntrb)((ring->index < ring->ntrb) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/usb/xhci.c", 1799, "ring->index < ring->ntrb"
));
1800
1801 /* Setup the link TRB after the previous TRB is done. */
1802 if (ring->index == 0) {
1803 lnk = &ring->trbs[ring->ntrb - 1];
1804 trb = &ring->trbs[ring->ntrb - 2];
1805
1806 bus_dmamap_sync(ring->dma.tag, ring->dma.map, TRBOFF(ring, lnk),(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(lnk) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02 | 0x08))
1807 sizeof(struct xhci_trb), BUS_DMASYNC_POSTREAD |(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(lnk) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02 | 0x08))
1808 BUS_DMASYNC_POSTWRITE)(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(lnk) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02 | 0x08));
1809
1810 lnk->trb_flags &= htole32(~XHCI_TRB_CHAIN)((__uint32_t)(~(1 << 4)));
1811 if (letoh32(trb->trb_flags)((__uint32_t)(trb->trb_flags)) & XHCI_TRB_CHAIN(1 << 4))
1812 lnk->trb_flags |= htole32(XHCI_TRB_CHAIN)((__uint32_t)((1 << 4)));
1813
1814 bus_dmamap_sync(ring->dma.tag, ring->dma.map, TRBOFF(ring, lnk),(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(lnk) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x04))
1815 sizeof(struct xhci_trb), BUS_DMASYNC_PREWRITE)(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(lnk) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x04));
1816
1817 lnk->trb_flags ^= htole32(XHCI_TRB_CYCLE)((__uint32_t)((1 << 0)));
1818
1819 bus_dmamap_sync(ring->dma.tag, ring->dma.map, TRBOFF(ring, lnk),(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(lnk) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x04))
1820 sizeof(struct xhci_trb), BUS_DMASYNC_PREWRITE)(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(lnk) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x04));
1821 }
1822
1823 trb = &ring->trbs[ring->index++];
1824 bus_dmamap_sync(ring->dma.tag, ring->dma.map, TRBOFF(ring, trb),(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(trb) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02 | 0x08))
1825 sizeof(struct xhci_trb), BUS_DMASYNC_POSTREAD |(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(trb) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02 | 0x08))
1826 BUS_DMASYNC_POSTWRITE)(*(ring->dma.tag)->_dmamap_sync)((ring->dma.tag), (ring
->dma.map), (((char *)(trb) - (char *)((ring)->trbs))),
(sizeof(struct xhci_trb)), (0x02 | 0x08));
1827
1828 /* Toggle cycle state of the link TRB and skip it. */
1829 if (ring->index == (ring->ntrb - 1)) {
1830 ring->index = 0;
1831 ring->toggle ^= 1;
1832 }
1833
1834 return (trb);
1835}
1836
1837struct xhci_trb *
1838xhci_xfer_get_trb(struct xhci_softc *sc, struct usbd_xfer *xfer,
1839 uint8_t *togglep, int last)
1840{
1841 struct xhci_pipe *xp = (struct xhci_pipe *)xfer->pipe;
1842 struct xhci_xfer *xx = (struct xhci_xfer *)xfer;
1843
1844 KASSERT(xp->free_trbs >= 1)((xp->free_trbs >= 1) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/usb/xhci.c", 1844, "xp->free_trbs >= 1"
));
1845 xp->free_trbs--;
1846 *togglep = xp->ring.toggle;
1847
1848 switch (last) {
1849 case -1: /* This will be a zero-length TD. */
1850 xp->pending_xfers[xp->ring.index] = NULL((void *)0);
1851 xx->zerotd += 1;
1852 break;
1853 case 0: /* This will be in a chain. */
1854 xp->pending_xfers[xp->ring.index] = xfer;
1855 xx->index = -2;
1856 xx->ntrb += 1;
1857 break;
1858 case 1: /* This will terminate a chain. */
1859 xp->pending_xfers[xp->ring.index] = xfer;
1860 xx->index = xp->ring.index;
1861 xx->ntrb += 1;
1862 break;
1863 }
1864
1865 xp->trb_processed[xp->ring.index] = TRB_PROCESSED_NO0;
1866
1867 return (xhci_ring_produce(sc, &xp->ring));
1868}
1869
1870int
1871xhci_command_submit(struct xhci_softc *sc, struct xhci_trb *trb0, int timeout)
1872{
1873 struct xhci_trb *trb;
1874 int s, error = 0;
1875
1876 KASSERT(timeout == 0 || sc->sc_cmd_trb == NULL)((timeout == 0 || sc->sc_cmd_trb == ((void *)0)) ? (void)0
: __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c", 1876
, "timeout == 0 || sc->sc_cmd_trb == NULL"));
1877
1878 trb0->trb_flags |= htole32(sc->sc_cmd_ring.toggle)((__uint32_t)(sc->sc_cmd_ring.toggle));
1879
1880 trb = xhci_ring_produce(sc, &sc->sc_cmd_ring);
1881 if (trb == NULL((void *)0))
1882 return (EAGAIN35);
1883 trb->trb_paddr = trb0->trb_paddr;
1884 trb->trb_status = trb0->trb_status;
1885 bus_dmamap_sync(sc->sc_cmd_ring.dma.tag, sc->sc_cmd_ring.dma.map,(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x04))
1886 TRBOFF(&sc->sc_cmd_ring, trb), sizeof(struct xhci_trb),(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x04))
1887 BUS_DMASYNC_PREWRITE)(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x04));
1888
1889 trb->trb_flags = trb0->trb_flags;
1890 bus_dmamap_sync(sc->sc_cmd_ring.dma.tag, sc->sc_cmd_ring.dma.map,(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x04))
1891 TRBOFF(&sc->sc_cmd_ring, trb), sizeof(struct xhci_trb),(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x04))
1892 BUS_DMASYNC_PREWRITE)(*(sc->sc_cmd_ring.dma.tag)->_dmamap_sync)((sc->sc_cmd_ring
.dma.tag), (sc->sc_cmd_ring.dma.map), (((char *)(trb) - (char
*)((&sc->sc_cmd_ring)->trbs))), (sizeof(struct xhci_trb
)), (0x04));
1893
1894 if (timeout == 0) {
1895 XDWRITE4(sc, XHCI_DOORBELL(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_door_off
+ ((0x0000 + (4 * (0))))), ((0))));
1896 return (0);
1897 }
1898
1899 rw_assert_wrlock(&sc->sc_cmd_lock);
1900
1901 s = splusb()splraise(0x2);
1902 sc->sc_cmd_trb = trb;
1903 XDWRITE4(sc, XHCI_DOORBELL(0), 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_door_off
+ ((0x0000 + (4 * (0))))), ((0))));
1904 error = tsleep_nsec(&sc->sc_cmd_trb, PZERO22, "xhcicmd", timeout);
1905 if (error) {
1906#ifdef XHCI_DEBUG
1907 printf("%s: tsleep() = %d\n", __func__, error);
1908 printf("cmd = %d ", XHCI_TRB_TYPE(letoh32(trb->trb_flags))(((((__uint32_t)(trb->trb_flags))) & 0xfc00) >> 10
));
1909 xhci_dump_trb(trb);
1910#endif
1911 KASSERT(sc->sc_cmd_trb == trb || sc->sc_cmd_trb == NULL)((sc->sc_cmd_trb == trb || sc->sc_cmd_trb == ((void *)0
)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c"
, 1911, "sc->sc_cmd_trb == trb || sc->sc_cmd_trb == NULL"
));
1912 /*
1913 * Just because the timeout expired this does not mean that the
1914 * TRB isn't active anymore! We could get an interrupt from
1915 * this TRB later on and then wonder what to do with it.
1916 * We'd rather abort it.
1917 */
1918 xhci_command_abort(sc);
1919 sc->sc_cmd_trb = NULL((void *)0);
1920 splx(s)spllower(s);
1921 return (error);
1922 }
1923 splx(s)spllower(s);
1924
1925 memcpy(trb0, &sc->sc_result_trb, sizeof(struct xhci_trb))__builtin_memcpy((trb0), (&sc->sc_result_trb), (sizeof
(struct xhci_trb)));
1926
1927 if (XHCI_TRB_GET_CODE(letoh32(trb0->trb_status))(((((__uint32_t)(trb0->trb_status))) >> 24) & 0xff
) == XHCI_CODE_SUCCESS1)
1928 return (0);
1929
1930#ifdef XHCI_DEBUG
1931 printf("%s: event error code=%d, result=%d \n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname),
1932 XHCI_TRB_GET_CODE(letoh32(trb0->trb_status))(((((__uint32_t)(trb0->trb_status))) >> 24) & 0xff
),
1933 XHCI_TRB_TYPE(letoh32(trb0->trb_flags))(((((__uint32_t)(trb0->trb_flags))) & 0xfc00) >>
10));
1934 xhci_dump_trb(trb0);
1935#endif
1936 return (EIO5);
1937}
1938
1939int
1940xhci_command_abort(struct xhci_softc *sc)
1941{
1942 uint32_t reg;
1943 int i;
1944
1945 reg = XOREAD4(sc, XHCI_CRCR_LO)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x18))));
1946 if ((reg & XHCI_CRCR_LO_CRR0x00000008) == 0)
1947 return (0);
1948
1949 XOWRITE4(sc, XHCI_CRCR_LO, reg | XHCI_CRCR_LO_CA)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x18)), ((reg | 0x00000004))));
1950 XOWRITE4(sc, XHCI_CRCR_HI, 0)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x1C)), ((0))));
1951
1952 for (i = 0; i < 2500; i++) {
1953 DELAY(100)(*delay_func)(100);
1954 reg = XOREAD4(sc, XHCI_CRCR_LO)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (0x18)))) & XHCI_CRCR_LO_CRR0x00000008;
1955 if (!reg)
1956 break;
1957 }
1958
1959 if (reg) {
1960 printf("%s: command ring abort timeout\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
1961 return (1);
1962 }
1963
1964 return (0);
1965}
1966
1967int
1968xhci_cmd_configure_ep(struct xhci_softc *sc, uint8_t slot, uint64_t addr)
1969{
1970 struct xhci_trb trb;
1971 int error;
1972
1973 DPRINTF(("%s: %s dev %u\n", DEVNAME(sc), __func__, slot));
1974
1975 trb.trb_paddr = htole64(addr)((__uint64_t)(addr));
1976 trb.trb_status = 0;
1977 trb.trb_flags = htole32(((__uint32_t)((((slot) & 0xff) << 24) | (12 <<
10)))
1978 XHCI_TRB_SET_SLOT(slot) | XHCI_CMD_CONFIG_EP((__uint32_t)((((slot) & 0xff) << 24) | (12 <<
10)))
1979 )((__uint32_t)((((slot) & 0xff) << 24) | (12 <<
10)));
1980
1981 rw_enter_write(&sc->sc_cmd_lock);
1982 error = xhci_command_submit(sc, &trb, XHCI_CMD_TIMEOUTMSEC_TO_NSEC(500));
1983 rw_exit_write(&sc->sc_cmd_lock);
1984 return (error);
1985}
1986
1987int
1988xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t slot, uint8_t dci)
1989{
1990 struct xhci_trb trb;
1991 int error;
1992
1993 DPRINTF(("%s: %s dev %u dci %u\n", DEVNAME(sc), __func__, slot, dci));
1994
1995 trb.trb_paddr = 0;
1996 trb.trb_status = 0;
1997 trb.trb_flags = htole32(((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (15 << 10)))
1998 XHCI_TRB_SET_SLOT(slot) | XHCI_TRB_SET_EP(dci) | XHCI_CMD_STOP_EP((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (15 << 10)))
1999 )((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (15 << 10)));
2000
2001 rw_enter_write(&sc->sc_cmd_lock);
2002 error = xhci_command_submit(sc, &trb, XHCI_CMD_TIMEOUTMSEC_TO_NSEC(500));
2003 rw_exit_write(&sc->sc_cmd_lock);
2004 return (error);
2005}
2006
2007void
2008xhci_cmd_reset_ep_async(struct xhci_softc *sc, uint8_t slot, uint8_t dci)
2009{
2010 struct xhci_trb trb;
2011
2012 DPRINTF(("%s: %s dev %u dci %u\n", DEVNAME(sc), __func__, slot, dci));
2013
2014 trb.trb_paddr = 0;
2015 trb.trb_status = 0;
2016 trb.trb_flags = htole32(((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (14 << 10)))
2017 XHCI_TRB_SET_SLOT(slot) | XHCI_TRB_SET_EP(dci) | XHCI_CMD_RESET_EP((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (14 << 10)))
2018 )((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (14 << 10)));
2019
2020 xhci_command_submit(sc, &trb, 0);
2021}
2022
2023void
2024xhci_cmd_set_tr_deq_async(struct xhci_softc *sc, uint8_t slot, uint8_t dci,
2025 uint64_t addr)
2026{
2027 struct xhci_trb trb;
2028
2029 DPRINTF(("%s: %s dev %u dci %u\n", DEVNAME(sc), __func__, slot, dci));
2030
2031 trb.trb_paddr = htole64(addr)((__uint64_t)(addr));
2032 trb.trb_status = 0;
2033 trb.trb_flags = htole32(((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (16 << 10)))
2034 XHCI_TRB_SET_SLOT(slot) | XHCI_TRB_SET_EP(dci) | XHCI_CMD_SET_TR_DEQ((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (16 << 10)))
2035 )((__uint32_t)((((slot) & 0xff) << 24) | (((dci) &
0x1f) << 16) | (16 << 10)));
2036
2037 xhci_command_submit(sc, &trb, 0);
2038}
2039
2040int
2041xhci_cmd_slot_control(struct xhci_softc *sc, uint8_t *slotp, int enable)
2042{
2043 struct xhci_trb trb;
2044 int error;
2045
2046 DPRINTF(("%s: %s\n", DEVNAME(sc), __func__));
2047
2048 trb.trb_paddr = 0;
2049 trb.trb_status = 0;
2050 if (enable)
2051 trb.trb_flags = htole32(XHCI_CMD_ENABLE_SLOT)((__uint32_t)((9 << 10)));
2052 else
2053 trb.trb_flags = htole32(((__uint32_t)((((*slotp) & 0xff) << 24) | (10 <<
10)))
2054 XHCI_TRB_SET_SLOT(*slotp) | XHCI_CMD_DISABLE_SLOT((__uint32_t)((((*slotp) & 0xff) << 24) | (10 <<
10)))
2055 )((__uint32_t)((((*slotp) & 0xff) << 24) | (10 <<
10)));
2056
2057 rw_enter_write(&sc->sc_cmd_lock);
2058 error = xhci_command_submit(sc, &trb, XHCI_CMD_TIMEOUTMSEC_TO_NSEC(500));
2059 rw_exit_write(&sc->sc_cmd_lock);
2060 if (error != 0)
2061 return (EIO5);
2062
2063 if (enable)
2064 *slotp = XHCI_TRB_GET_SLOT(letoh32(trb.trb_flags))(((((__uint32_t)(trb.trb_flags))) >> 24) & 0xff);
2065
2066 return (0);
2067}
2068
2069int
2070xhci_cmd_set_address(struct xhci_softc *sc, uint8_t slot, uint64_t addr,
2071 uint32_t bsr)
2072{
2073 struct xhci_trb trb;
2074 int error;
2075
2076 DPRINTF(("%s: %s BSR=%u\n", DEVNAME(sc), __func__, bsr ? 1 : 0));
2077
2078 trb.trb_paddr = htole64(addr)((__uint64_t)(addr));
2079 trb.trb_status = 0;
2080 trb.trb_flags = htole32(((__uint32_t)((((slot) & 0xff) << 24) | (11 <<
10) | bsr))
2081 XHCI_TRB_SET_SLOT(slot) | XHCI_CMD_ADDRESS_DEVICE | bsr((__uint32_t)((((slot) & 0xff) << 24) | (11 <<
10) | bsr))
2082 )((__uint32_t)((((slot) & 0xff) << 24) | (11 <<
10) | bsr));
2083
2084 rw_enter_write(&sc->sc_cmd_lock);
2085 error = xhci_command_submit(sc, &trb, XHCI_CMD_TIMEOUTMSEC_TO_NSEC(500));
2086 rw_exit_write(&sc->sc_cmd_lock);
2087 return (error);
2088}
2089
2090#ifdef XHCI_DEBUG
2091int
2092xhci_cmd_noop(struct xhci_softc *sc)
2093{
2094 struct xhci_trb trb;
2095 int error;
2096
2097 DPRINTF(("%s: %s\n", DEVNAME(sc), __func__));
2098
2099 trb.trb_paddr = 0;
2100 trb.trb_status = 0;
2101 trb.trb_flags = htole32(XHCI_CMD_NOOP)((__uint32_t)((23 << 10)));
2102
2103 rw_enter_write(&sc->sc_cmd_lock);
2104 error = xhci_command_submit(sc, &trb, XHCI_CMD_TIMEOUTMSEC_TO_NSEC(500));
2105 rw_exit_write(&sc->sc_cmd_lock);
2106 return (error);
2107}
2108#endif
2109
2110int
2111xhci_softdev_alloc(struct xhci_softc *sc, uint8_t slot)
2112{
2113 struct xhci_soft_dev *sdev = &sc->sc_sdevs[slot];
2114 int i, error;
2115 uint8_t *kva;
2116
2117 /*
2118 * Setup input context. Even with 64 byte context size, it
2119 * fits into the smallest supported page size, so use that.
2120 */
2121 error = usbd_dma_contig_alloc(&sc->sc_bus, &sdev->ictx_dma,
2122 (void **)&kva, sc->sc_pagesize, XHCI_ICTX_ALIGN64, sc->sc_pagesize);
2123 if (error)
2124 return (ENOMEM12);
2125
2126 sdev->input_ctx = (struct xhci_inctx *)kva;
2127 sdev->slot_ctx = (struct xhci_sctx *)(kva + sc->sc_ctxsize);
2128 for (i = 0; i < 31; i++)
2129 sdev->ep_ctx[i] =
2130 (struct xhci_epctx *)(kva + (i + 2) * sc->sc_ctxsize);
2131
2132 DPRINTF(("%s: dev %d, input=%p slot=%p ep0=%p\n", DEVNAME(sc),
2133 slot, sdev->input_ctx, sdev->slot_ctx, sdev->ep_ctx[0]));
2134
2135 /* Setup output context */
2136 error = usbd_dma_contig_alloc(&sc->sc_bus, &sdev->octx_dma, NULL((void *)0),
2137 sc->sc_pagesize, XHCI_OCTX_ALIGN32, sc->sc_pagesize);
2138 if (error) {
2139 usbd_dma_contig_free(&sc->sc_bus, &sdev->ictx_dma);
2140 return (ENOMEM12);
2141 }
2142
2143 memset(&sdev->pipes, 0, sizeof(sdev->pipes))__builtin_memset((&sdev->pipes), (0), (sizeof(sdev->
pipes)));
2144
2145 DPRINTF(("%s: dev %d, setting DCBAA to 0x%016llx\n", DEVNAME(sc),
2146 slot, (long long)sdev->octx_dma.paddr));
2147
2148 sc->sc_dcbaa.segs[slot] = htole64(sdev->octx_dma.paddr)((__uint64_t)(sdev->octx_dma.paddr));
2149 bus_dmamap_sync(sc->sc_dcbaa.dma.tag, sc->sc_dcbaa.dma.map,(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (slot * sizeof(uint64_t
)), (sizeof(uint64_t)), (0x01 | 0x04))
2150 slot * sizeof(uint64_t), sizeof(uint64_t), BUS_DMASYNC_PREREAD |(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (slot * sizeof(uint64_t
)), (sizeof(uint64_t)), (0x01 | 0x04))
2151 BUS_DMASYNC_PREWRITE)(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (slot * sizeof(uint64_t
)), (sizeof(uint64_t)), (0x01 | 0x04));
2152
2153 return (0);
2154}
2155
2156void
2157xhci_softdev_free(struct xhci_softc *sc, uint8_t slot)
2158{
2159 struct xhci_soft_dev *sdev = &sc->sc_sdevs[slot];
2160
2161 sc->sc_dcbaa.segs[slot] = 0;
2162 bus_dmamap_sync(sc->sc_dcbaa.dma.tag, sc->sc_dcbaa.dma.map,(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (slot * sizeof(uint64_t
)), (sizeof(uint64_t)), (0x01 | 0x04))
2163 slot * sizeof(uint64_t), sizeof(uint64_t), BUS_DMASYNC_PREREAD |(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (slot * sizeof(uint64_t
)), (sizeof(uint64_t)), (0x01 | 0x04))
2164 BUS_DMASYNC_PREWRITE)(*(sc->sc_dcbaa.dma.tag)->_dmamap_sync)((sc->sc_dcbaa
.dma.tag), (sc->sc_dcbaa.dma.map), (slot * sizeof(uint64_t
)), (sizeof(uint64_t)), (0x01 | 0x04));
2165
2166 usbd_dma_contig_free(&sc->sc_bus, &sdev->octx_dma);
2167 usbd_dma_contig_free(&sc->sc_bus, &sdev->ictx_dma);
2168
2169 memset(sdev, 0, sizeof(struct xhci_soft_dev))__builtin_memset((sdev), (0), (sizeof(struct xhci_soft_dev)));
2170}
2171
2172/* Root hub descriptors. */
2173const usb_device_descriptor_t xhci_devd = {
2174 USB_DEVICE_DESCRIPTOR_SIZE18,
2175 UDESC_DEVICE0x01, /* type */
2176 {0x00, 0x03}, /* USB version */
2177 UDCLASS_HUB0x09, /* class */
2178 UDSUBCLASS_HUB0x00, /* subclass */
2179 UDPROTO_HSHUBSTT0x01, /* protocol */
2180 9, /* max packet */
2181 {0},{0},{0x00,0x01}, /* device id */
2182 1,2,0, /* string indexes */
2183 1 /* # of configurations */
2184};
2185
2186const usb_config_descriptor_t xhci_confd = {
2187 USB_CONFIG_DESCRIPTOR_SIZE9,
2188 UDESC_CONFIG0x02,
2189 {USB_CONFIG_DESCRIPTOR_SIZE9 +
2190 USB_INTERFACE_DESCRIPTOR_SIZE9 +
2191 USB_ENDPOINT_DESCRIPTOR_SIZE7},
2192 1,
2193 1,
2194 0,
2195 UC_BUS_POWERED0x80 | UC_SELF_POWERED0x40,
2196 0 /* max power */
2197};
2198
2199const usb_interface_descriptor_t xhci_ifcd = {
2200 USB_INTERFACE_DESCRIPTOR_SIZE9,
2201 UDESC_INTERFACE0x04,
2202 0,
2203 0,
2204 1,
2205 UICLASS_HUB0x09,
2206 UISUBCLASS_HUB0,
2207 UIPROTO_HSHUBSTT0,
2208 0
2209};
2210
2211const usb_endpoint_descriptor_t xhci_endpd = {
2212 USB_ENDPOINT_DESCRIPTOR_SIZE7,
2213 UDESC_ENDPOINT0x05,
2214 UE_DIR_IN0x80 | XHCI_INTR_ENDPT1,
2215 UE_INTERRUPT0x03,
2216 {2, 0}, /* max 15 ports */
2217 255
2218};
2219
2220const usb_endpoint_ss_comp_descriptor_t xhci_endpcd = {
2221 USB_ENDPOINT_SS_COMP_DESCRIPTOR_SIZE6,
2222 UDESC_ENDPOINT_SS_COMP0x30,
2223 0,
2224 0,
2225 {0, 0}
2226};
2227
2228const usb_hub_descriptor_t xhci_hubd = {
2229 USB_HUB_DESCRIPTOR_SIZE8,
2230 UDESC_SS_HUB0x2A,
2231 0,
2232 {0,0},
2233 0,
2234 0,
2235 {0},
2236};
2237
2238void
2239xhci_abort_xfer(struct usbd_xfer *xfer, usbd_status status)
2240{
2241 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
2242 struct xhci_pipe *xp = (struct xhci_pipe *)xfer->pipe;
2243 int error;
2244
2245 splsoftassert(IPL_SOFTUSB)do { if (splassert_ctl > 0) { splassert_check(0x2, __func__
); } } while (0);
2246
2247 DPRINTF(("%s: xfer=%p status=%s err=%s actlen=%d len=%d idx=%d\n",
2248 __func__, xfer, usbd_errstr(xfer->status), usbd_errstr(status),
2249 xfer->actlen, xfer->length, ((struct xhci_xfer *)xfer)->index));
2250
2251 /* XXX The stack should not call abort() in this case. */
2252 if (sc->sc_bus.dying || xfer->status == USBD_NOT_STARTED) {
2253 xfer->status = status;
2254 timeout_del(&xfer->timeout_handle);
2255 usb_rem_task(xfer->device, &xfer->abort_task);
2256 usb_transfer_complete(xfer);
2257 return;
2258 }
2259
2260 /* Transfer is already done. */
2261 if (xfer->status != USBD_IN_PROGRESS) {
2262 DPRINTF(("%s: already done \n", __func__));
2263 return;
2264 }
2265
2266 /* Prevent any timeout to kick in. */
2267 timeout_del(&xfer->timeout_handle);
2268 usb_rem_task(xfer->device, &xfer->abort_task);
2269
2270 /* Indicate that we are aborting this transfer. */
2271 xp->halted = status;
2272 xp->aborted_xfer = xfer;
2273
2274 /* Stop the endpoint and wait until the hardware says so. */
2275 if (xhci_cmd_stop_ep(sc, xp->slot, xp->dci)) {
2276 DPRINTF(("%s: error stopping endpoint\n", DEVNAME(sc)));
2277 /* Assume the device is gone. */
2278 xp->halted = 0;
2279 xp->aborted_xfer = NULL((void *)0);
2280 xfer->status = status;
2281 usb_transfer_complete(xfer);
2282 return;
2283 }
2284
2285 /*
2286 * The transfer was already completed when we stopped the
2287 * endpoint, no need to move the dequeue pointer past its
2288 * TRBs.
2289 */
2290 if (xp->aborted_xfer == NULL((void *)0)) {
2291 DPRINTF(("%s: done before stopping the endpoint\n", __func__));
2292 xp->halted = 0;
2293 return;
2294 }
2295
2296 /*
2297 * At this stage the endpoint has been stopped, so update its
2298 * dequeue pointer past the last TRB of the transfer.
2299 *
2300 * Note: This assumes that only one transfer per endpoint has
2301 * pending TRBs on the ring.
2302 */
2303 xhci_cmd_set_tr_deq_async(sc, xp->slot, xp->dci,
2304 DEQPTR(xp->ring)((xp->ring).dma.paddr + (sizeof(struct xhci_trb) * (xp->
ring).index)) | xp->ring.toggle);
2305 error = tsleep_nsec(xp, PZERO22, "xhciab", XHCI_CMD_TIMEOUTMSEC_TO_NSEC(500));
2306 if (error)
2307 printf("%s: timeout aborting transfer\n", DEVNAME(sc)((sc)->sc_bus.bdev.dv_xname));
2308}
2309
2310void
2311xhci_timeout(void *addr)
2312{
2313 struct usbd_xfer *xfer = addr;
2314 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
2315
2316 if (sc->sc_bus.dying) {
2317 xhci_timeout_task(addr);
2318 return;
2319 }
2320
2321 usb_init_task(&xfer->abort_task, xhci_timeout_task, addr,((&xfer->abort_task)->fun = (xhci_timeout_task), (&
xfer->abort_task)->arg = (addr), (&xfer->abort_task
)->type = (2), (&xfer->abort_task)->state = 0x0)
2322 USB_TASK_TYPE_ABORT)((&xfer->abort_task)->fun = (xhci_timeout_task), (&
xfer->abort_task)->arg = (addr), (&xfer->abort_task
)->type = (2), (&xfer->abort_task)->state = 0x0);
2323 usb_add_task(xfer->device, &xfer->abort_task);
2324}
2325
2326void
2327xhci_timeout_task(void *addr)
2328{
2329 struct usbd_xfer *xfer = addr;
2330 int s;
2331
2332 s = splusb()splraise(0x2);
2333 xhci_abort_xfer(xfer, USBD_TIMEOUT);
2334 splx(s)spllower(s);
2335}
2336
2337usbd_status
2338xhci_root_ctrl_transfer(struct usbd_xfer *xfer)
2339{
2340 usbd_status err;
2341
2342 err = usb_insert_transfer(xfer);
2343 if (err)
2344 return (err);
2345
2346 return (xhci_root_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)((&xfer->pipe->queue)->sqh_first)));
2347}
2348
2349usbd_status
2350xhci_root_ctrl_start(struct usbd_xfer *xfer)
2351{
2352 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
2353 usb_port_status_t ps;
2354 usb_device_request_t *req;
2355 void *buf = NULL((void *)0);
1
'buf' initialized to a null pointer value
→
2356 usb_device_descriptor_t devd;
2357 usb_hub_descriptor_t hubd;
2358 usbd_status err;
2359 int s, len, value, index;
2360 int l, totlen = 0;
2361 int port, i;
2362 uint32_t v;
2363
2364 KASSERT(xfer->rqflags & URQ_REQUEST)((xfer->rqflags & 0x01) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/usb/xhci.c", 2364, "xfer->rqflags & URQ_REQUEST"
));
2
←
Assuming the condition is true
→
3
←
'?' condition is true
→
2365
2366 if (sc->sc_bus.dying)
4
←
Assuming field 'dying' is 0
→
5
←
Taking false branch
→
2367 return (USBD_IOERROR);
2368
2369 req = &xfer->request;
2370
2371 DPRINTFN(4,("%s: type=0x%02x request=%02x\n", __func__,
2372 req->bmRequestType, req->bRequest));
2373
2374 len = UGETW(req->wLength)(*(u_int16_t *)(req->wLength));
2375 value = UGETW(req->wValue)(*(u_int16_t *)(req->wValue));
2376 index = UGETW(req->wIndex)(*(u_int16_t *)(req->wIndex));
2377
2378 if (len != 0)
6
←
Assuming 'len' is equal to 0
→
7
←
Taking false branch
→
2379 buf = KERNADDR(&xfer->dmabuf, 0)((void *)((char *)((&xfer->dmabuf)->block->kaddr
+ (&xfer->dmabuf)->offs) + (0)));
2380
2381#define C(x,y)((x) | ((y) << 8)) ((x) | ((y) << 8))
2382 switch(C(req->bRequest, req->bmRequestType)((req->bRequest) | ((req->bmRequestType) << 8))) {
8
←
Control jumps to 'case 32774:' at line 2397
→
2383 case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE)((0x01) | (((0x00 | 0x00 | 0x00)) << 8)):
2384 case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE)((0x01) | (((0x00 | 0x00 | 0x01)) << 8)):
2385 case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT)((0x01) | (((0x00 | 0x00 | 0x02)) << 8)):
2386 /*
2387 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2388 * for the integrated root hub.
2389 */
2390 break;
2391 case C(UR_GET_CONFIG, UT_READ_DEVICE)((0x08) | (((0x80 | 0x00 | 0x00)) << 8)):
2392 if (len > 0) {
2393 *(uint8_t *)buf = sc->sc_conf;
2394 totlen = 1;
2395 }
2396 break;
2397 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE)((0x06) | (((0x80 | 0x00 | 0x00)) << 8)):
2398 DPRINTFN(8,("xhci_root_ctrl_start: wValue=0x%04x\n", value));
2399 switch(value >> 8) {
9
←
Control jumps to 'case 2:' at line 2415
→
2400 case UDESC_DEVICE0x01:
2401 if ((value & 0xff) != 0) {
2402 err = USBD_IOERROR;
2403 goto ret;
2404 }
2405 devd = xhci_devd;
2406 USETW(devd.idVendor, sc->sc_id_vendor)(*(u_int16_t *)(devd.idVendor) = (sc->sc_id_vendor));
2407 totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE18);
2408 memcpy(buf, &devd, l)__builtin_memcpy((buf), (&devd), (l));
2409 break;
2410 /*
2411 * We can't really operate at another speed, but the spec says
2412 * we need this descriptor.
2413 */
2414 case UDESC_OTHER_SPEED_CONFIGURATION0x07:
2415 case UDESC_CONFIG0x02:
2416 if ((value & 0xff) != 0) {
10
←
Assuming the condition is false
→
11
←
Taking false branch
→
2417 err = USBD_IOERROR;
2418 goto ret;
2419 }
2420 totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE9);
2421 memcpy(buf, &xhci_confd, l)__builtin_memcpy((buf), (&xhci_confd), (l));
2422 ((usb_config_descriptor_t *)buf)->bDescriptorType =
12
←
Access to field 'bDescriptorType' results in a dereference of a null pointer (loaded from variable 'buf')
2423 value >> 8;
2424 buf = (char *)buf + l;
2425 len -= l;
2426 l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE9);
2427 totlen += l;
2428 memcpy(buf, &xhci_ifcd, l)__builtin_memcpy((buf), (&xhci_ifcd), (l));
2429 buf = (char *)buf + l;
2430 len -= l;
2431 l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE7);
2432 totlen += l;
2433 memcpy(buf, &xhci_endpd, l)__builtin_memcpy((buf), (&xhci_endpd), (l));
2434 break;
2435 case UDESC_STRING0x03:
2436 if (len == 0)
2437 break;
2438 *(u_int8_t *)buf = 0;
2439 totlen = 1;
2440 switch (value & 0xff) {
2441 case 0: /* Language table */
2442 totlen = usbd_str(buf, len, "\001");
2443 break;
2444 case 1: /* Vendor */
2445 totlen = usbd_str(buf, len, sc->sc_vendor);
2446 break;
2447 case 2: /* Product */
2448 totlen = usbd_str(buf, len, "xHCI root hub");
2449 break;
2450 }
2451 break;
2452 default:
2453 err = USBD_IOERROR;
2454 goto ret;
2455 }
2456 break;
2457 case C(UR_GET_INTERFACE, UT_READ_INTERFACE)((0x0a) | (((0x80 | 0x00 | 0x01)) << 8)):
2458 if (len > 0) {
2459 *(uint8_t *)buf = 0;
2460 totlen = 1;
2461 }
2462 break;
2463 case C(UR_GET_STATUS, UT_READ_DEVICE)((0x00) | (((0x80 | 0x00 | 0x00)) << 8)):
2464 if (len > 1) {
2465 USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED)(*(u_int16_t *)(((usb_status_t *)buf)->wStatus) = (0x0001)
);
2466 totlen = 2;
2467 }
2468 break;
2469 case C(UR_GET_STATUS, UT_READ_INTERFACE)((0x00) | (((0x80 | 0x00 | 0x01)) << 8)):
2470 case C(UR_GET_STATUS, UT_READ_ENDPOINT)((0x00) | (((0x80 | 0x00 | 0x02)) << 8)):
2471 if (len > 1) {
2472 USETW(((usb_status_t *)buf)->wStatus, 0)(*(u_int16_t *)(((usb_status_t *)buf)->wStatus) = (0));
2473 totlen = 2;
2474 }
2475 break;
2476 case C(UR_SET_ADDRESS, UT_WRITE_DEVICE)((0x05) | (((0x00 | 0x00 | 0x00)) << 8)):
2477 if (value >= USB_MAX_DEVICES128) {
2478 err = USBD_IOERROR;
2479 goto ret;
2480 }
2481 break;
2482 case C(UR_SET_CONFIG, UT_WRITE_DEVICE)((0x09) | (((0x00 | 0x00 | 0x00)) << 8)):
2483 if (value != 0 && value != 1) {
2484 err = USBD_IOERROR;
2485 goto ret;
2486 }
2487 sc->sc_conf = value;
2488 break;
2489 case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE)((0x07) | (((0x00 | 0x00 | 0x00)) << 8)):
2490 break;
2491 case C(UR_SET_FEATURE, UT_WRITE_DEVICE)((0x03) | (((0x00 | 0x00 | 0x00)) << 8)):
2492 case C(UR_SET_FEATURE, UT_WRITE_INTERFACE)((0x03) | (((0x00 | 0x00 | 0x01)) << 8)):
2493 case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT)((0x03) | (((0x00 | 0x00 | 0x02)) << 8)):
2494 err = USBD_IOERROR;
2495 goto ret;
2496 case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE)((0x0b) | (((0x00 | 0x00 | 0x01)) << 8)):
2497 break;
2498 case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT)((0x0c) | (((0x00 | 0x00 | 0x02)) << 8)):
2499 break;
2500 /* Hub requests */
2501 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE)((0x01) | (((0x00 | 0x20 | 0x00)) << 8)):
2502 break;
2503 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER)((0x01) | (((0x00 | 0x20 | 0x03)) << 8)):
2504 DPRINTFN(8, ("xhci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
2505 "port=%d feature=%d\n", index, value));
2506 if (index < 1 || index > sc->sc_noport) {
2507 err = USBD_IOERROR;
2508 goto ret;
2509 }
2510 port = XHCI_PORTSC(index)(0x3f0 + (0x10 * (index)));
2511 v = XOREAD4(sc, port)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)))) & ~XHCI_PS_CLEAR0x80ff01ffu;
2512 switch (value) {
2513 case UHF_PORT_ENABLE1:
2514 XOWRITE4(sc, port, v | XHCI_PS_PED)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00000002))));
2515 break;
2516 case UHF_PORT_SUSPEND2:
2517 /* TODO */
2518 break;
2519 case UHF_PORT_POWER8:
2520 XOWRITE4(sc, port, v & ~XHCI_PS_PP)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v & ~0x00000200))));
2521 break;
2522 case UHF_PORT_INDICATOR22:
2523 XOWRITE4(sc, port, v & ~XHCI_PS_SET_PIC(3))(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v & ~(((3) & 0x3) << 14)))));
2524 break;
2525 case UHF_C_PORT_CONNECTION16:
2526 XOWRITE4(sc, port, v | XHCI_PS_CSC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00020000))));
2527 break;
2528 case UHF_C_PORT_ENABLE17:
2529 XOWRITE4(sc, port, v | XHCI_PS_PEC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00040000))));
2530 break;
2531 case UHF_C_PORT_SUSPEND18:
2532 case UHF_C_PORT_LINK_STATE25:
2533 XOWRITE4(sc, port, v | XHCI_PS_PLC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00400000))));
2534 break;
2535 case UHF_C_PORT_OVER_CURRENT19:
2536 XOWRITE4(sc, port, v | XHCI_PS_OCC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00100000))));
2537 break;
2538 case UHF_C_PORT_RESET20:
2539 XOWRITE4(sc, port, v | XHCI_PS_PRC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00200000))));
2540 break;
2541 case UHF_C_BH_PORT_RESET29:
2542 XOWRITE4(sc, port, v | XHCI_PS_WRC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00080000))));
2543 break;
2544 default:
2545 err = USBD_IOERROR;
2546 goto ret;
2547 }
2548 break;
2549
2550 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE)((0x06) | (((0x80 | 0x20 | 0x00)) << 8)):
2551 if (len == 0)
2552 break;
2553 if ((value & 0xff) != 0) {
2554 err = USBD_IOERROR;
2555 goto ret;
2556 }
2557 v = XREAD4(sc, XHCI_HCCPARAMS)(((sc)->iot)->read_4(((sc)->ioh), ((0x10))));
2558 hubd = xhci_hubd;
2559 hubd.bNbrPorts = sc->sc_noport;
2560 USETW(hubd.wHubCharacteristics,(*(u_int16_t *)(hubd.wHubCharacteristics) = (((((v) >> 3
) & 0x1) ? 0x0001 : 0x0000) | ((((v) >> 4) & 0x1
) ? 0x0080 : 0)))
2561 (XHCI_HCC_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_GANGED) |(*(u_int16_t *)(hubd.wHubCharacteristics) = (((((v) >> 3
) & 0x1) ? 0x0001 : 0x0000) | ((((v) >> 4) & 0x1
) ? 0x0080 : 0)))
2562 (XHCI_HCC_PIND(v) ? UHD_PORT_IND : 0))(*(u_int16_t *)(hubd.wHubCharacteristics) = (((((v) >> 3
) & 0x1) ? 0x0001 : 0x0000) | ((((v) >> 4) & 0x1
) ? 0x0080 : 0)));
2563 hubd.bPwrOn2PwrGood = 10; /* xHCI section 5.4.9 */
2564 for (i = 1; i <= sc->sc_noport; i++) {
2565 v = XOREAD4(sc, XHCI_PORTSC(i))(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ ((0x3f0 + (0x10 * (i)))))));
2566 if (v & XHCI_PS_DR0x40000000)
2567 hubd.DeviceRemovable[i / 8] |= 1U << (i % 8);
2568 }
2569 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE8 + i;
2570 l = min(len, hubd.bDescLength);
2571 totlen = l;
2572 memcpy(buf, &hubd, l)__builtin_memcpy((buf), (&hubd), (l));
2573 break;
2574 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE)((0x00) | (((0x80 | 0x20 | 0x00)) << 8)):
2575 if (len != 16) {
2576 err = USBD_IOERROR;
2577 goto ret;
2578 }
2579 memset(buf, 0, len)__builtin_memset((buf), (0), (len));
2580 totlen = len;
2581 break;
2582 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER)((0x00) | (((0x80 | 0x20 | 0x03)) << 8)):
2583 DPRINTFN(8,("xhci_root_ctrl_start: get port status i=%d\n",
2584 index));
2585 if (index < 1 || index > sc->sc_noport) {
2586 err = USBD_IOERROR;
2587 goto ret;
2588 }
2589 if (len != 4) {
2590 err = USBD_IOERROR;
2591 goto ret;
2592 }
2593 v = XOREAD4(sc, XHCI_PORTSC(index))(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ ((0x3f0 + (0x10 * (index)))))));
2594 DPRINTFN(8,("xhci_root_ctrl_start: port status=0x%04x\n", v));
2595 i = UPS_PORT_LS_SET(XHCI_PS_GET_PLS(v))((((((v) >> 5) & 0xf)) & 0xf) << 5);
2596 switch (XHCI_PS_SPEED(v)(((v) >> 10) & 0xf)) {
2597 case XHCI_SPEED_FULL1:
2598 i |= UPS_FULL_SPEED0x0000;
2599 break;
2600 case XHCI_SPEED_LOW2:
2601 i |= UPS_LOW_SPEED0x0200;
2602 break;
2603 case XHCI_SPEED_HIGH3:
2604 i |= UPS_HIGH_SPEED0x0400;
2605 break;
2606 case XHCI_SPEED_SUPER4:
2607 default:
2608 break;
2609 }
2610 if (v & XHCI_PS_CCS0x00000001) i |= UPS_CURRENT_CONNECT_STATUS0x0001;
2611 if (v & XHCI_PS_PED0x00000002) i |= UPS_PORT_ENABLED0x0002;
2612 if (v & XHCI_PS_OCA0x00000008) i |= UPS_OVERCURRENT_INDICATOR0x0008;
2613 if (v & XHCI_PS_PR0x00000010) i |= UPS_RESET0x0010;
2614 if (v & XHCI_PS_PP0x00000200) {
2615 if (XHCI_PS_SPEED(v)(((v) >> 10) & 0xf) >= XHCI_SPEED_FULL1 &&
2616 XHCI_PS_SPEED(v)(((v) >> 10) & 0xf) <= XHCI_SPEED_HIGH3)
2617 i |= UPS_PORT_POWER0x0100;
2618 else
2619 i |= UPS_PORT_POWER_SS0x0200;
2620 }
2621 USETW(ps.wPortStatus, i)(*(u_int16_t *)(ps.wPortStatus) = (i));
2622 i = 0;
2623 if (v & XHCI_PS_CSC0x00020000) i |= UPS_C_CONNECT_STATUS0x0001;
2624 if (v & XHCI_PS_PEC0x00040000) i |= UPS_C_PORT_ENABLED0x0002;
2625 if (v & XHCI_PS_OCC0x00100000) i |= UPS_C_OVERCURRENT_INDICATOR0x0008;
2626 if (v & XHCI_PS_PRC0x00200000) i |= UPS_C_PORT_RESET0x0010;
2627 if (v & XHCI_PS_WRC0x00080000) i |= UPS_C_BH_PORT_RESET0x0020;
2628 if (v & XHCI_PS_PLC0x00400000) i |= UPS_C_PORT_LINK_STATE0x0040;
2629 if (v & XHCI_PS_CEC0x00800000) i |= UPS_C_PORT_CONFIG_ERROR0x0080;
2630 USETW(ps.wPortChange, i)(*(u_int16_t *)(ps.wPortChange) = (i));
2631 l = min(len, sizeof ps);
2632 memcpy(buf, &ps, l)__builtin_memcpy((buf), (&ps), (l));
2633 totlen = l;
2634 break;
2635 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE)((0x07) | (((0x00 | 0x20 | 0x00)) << 8)):
2636 err = USBD_IOERROR;
2637 goto ret;
2638 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE)((0x03) | (((0x00 | 0x20 | 0x00)) << 8)):
2639 break;
2640 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER)((0x03) | (((0x00 | 0x20 | 0x03)) << 8)):
2641
2642 i = index >> 8;
2643 index &= 0x00ff;
2644
2645 if (index < 1 || index > sc->sc_noport) {
2646 err = USBD_IOERROR;
2647 goto ret;
2648 }
2649 port = XHCI_PORTSC(index)(0x3f0 + (0x10 * (index)));
2650 v = XOREAD4(sc, port)(((sc)->iot)->read_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)))) & ~XHCI_PS_CLEAR0x80ff01ffu;
2651
2652 switch (value) {
2653 case UHF_PORT_ENABLE1:
2654 XOWRITE4(sc, port, v | XHCI_PS_PED)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00000002))));
2655 break;
2656 case UHF_PORT_SUSPEND2:
2657 DPRINTFN(6, ("suspend port %u (LPM=%u)\n", index, i));
2658 if (XHCI_PS_SPEED(v)(((v) >> 10) & 0xf) == XHCI_SPEED_SUPER4) {
2659 err = USBD_IOERROR;
2660 goto ret;
2661 }
2662 XOWRITE4(sc, port, v |(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | (((i ? 2 : 3) & 0xf) << 5) | 0x00010000
))))
2663 XHCI_PS_SET_PLS(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | (((i ? 2 : 3) & 0xf) << 5) | 0x00010000
))));
2664 break;
2665 case UHF_PORT_RESET4:
2666 DPRINTFN(6, ("reset port %d\n", index));
2667 XOWRITE4(sc, port, v | XHCI_PS_PR)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00000010))));
2668 break;
2669 case UHF_PORT_POWER8:
2670 DPRINTFN(3, ("set port power %d\n", index));
2671 XOWRITE4(sc, port, v | XHCI_PS_PP)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00000200))));
2672 break;
2673 case UHF_PORT_INDICATOR22:
2674 DPRINTFN(3, ("set port indicator %d\n", index));
2675
2676 v &= ~XHCI_PS_SET_PIC(3)(((3) & 0x3) << 14);
2677 v |= XHCI_PS_SET_PIC(1)(((1) & 0x3) << 14);
2678
2679 XOWRITE4(sc, port, v)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v))));
2680 break;
2681 case UHF_C_PORT_RESET20:
2682 XOWRITE4(sc, port, v | XHCI_PS_PRC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00200000))));
2683 break;
2684 case UHF_C_BH_PORT_RESET29:
2685 XOWRITE4(sc, port, v | XHCI_PS_WRC)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_oper_off
+ (port)), ((v | 0x00080000))));
2686 break;
2687 default:
2688 err = USBD_IOERROR;
2689 goto ret;
2690 }
2691 break;
2692 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER)((0x08) | (((0x00 | 0x20 | 0x03)) << 8)):
2693 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER)((0x09) | (((0x00 | 0x20 | 0x03)) << 8)):
2694 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER)((0x0a) | (((0x80 | 0x20 | 0x03)) << 8)):
2695 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER)((0x0b) | (((0x00 | 0x20 | 0x03)) << 8)):
2696 break;
2697 default:
2698 err = USBD_IOERROR;
2699 goto ret;
2700 }
2701 xfer->actlen = totlen;
2702 err = USBD_NORMAL_COMPLETION;
2703ret:
2704 xfer->status = err;
2705 s = splusb()splraise(0x2);
2706 usb_transfer_complete(xfer);
2707 splx(s)spllower(s);
2708 return (err);
2709}
2710
2711
2712void
2713xhci_noop(struct usbd_xfer *xfer)
2714{
2715}
2716
2717
2718usbd_status
2719xhci_root_intr_transfer(struct usbd_xfer *xfer)
2720{
2721 usbd_status err;
2722
2723 err = usb_insert_transfer(xfer);
2724 if (err)
2725 return (err);
2726
2727 return (xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)((&xfer->pipe->queue)->sqh_first)));
2728}
2729
2730usbd_status
2731xhci_root_intr_start(struct usbd_xfer *xfer)
2732{
2733 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
2734
2735 if (sc->sc_bus.dying)
2736 return (USBD_IOERROR);
2737
2738 sc->sc_intrxfer = xfer;
2739
2740 return (USBD_IN_PROGRESS);
2741}
2742
2743void
2744xhci_root_intr_abort(struct usbd_xfer *xfer)
2745{
2746 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
2747 int s;
2748
2749 sc->sc_intrxfer = NULL((void *)0);
2750
2751 xfer->status = USBD_CANCELLED;
2752 s = splusb()splraise(0x2);
2753 usb_transfer_complete(xfer);
2754 splx(s)spllower(s);
2755}
2756
2757void
2758xhci_root_intr_done(struct usbd_xfer *xfer)
2759{
2760}
2761
2762/*
2763 * Number of packets remaining in the TD after the corresponding TRB.
2764 *
2765 * Section 4.11.2.4 of xHCI specification r1.1.
2766 */
2767static inline uint32_t
2768xhci_xfer_tdsize(struct usbd_xfer *xfer, uint32_t remain, uint32_t len)
2769{
2770 uint32_t npkt, mps = UGETW(xfer->pipe->endpoint->edesc->wMaxPacketSize)(*(u_int16_t *)(xfer->pipe->endpoint->edesc->wMaxPacketSize
));
2771
2772 if (len == 0)
2773 return XHCI_TRB_TDREM(0)(((0) & 0x1f) << 17);
2774
2775 npkt = howmany(remain - len, UE_GET_SIZE(mps))(((remain - len) + ((((mps) & 0x7ff)) - 1)) / (((mps) &
0x7ff)));
2776 if (npkt > 31)
2777 npkt = 31;
2778
2779 return XHCI_TRB_TDREM(npkt)(((npkt) & 0x1f) << 17);
2780}
2781
2782/*
2783 * Transfer Burst Count (TBC) and Transfer Last Burst Packet Count (TLBPC).
2784 *
2785 * Section 4.11.2.3 of xHCI specification r1.1.
2786 */
2787static inline uint32_t
2788xhci_xfer_tbc(struct usbd_xfer *xfer, uint32_t len, uint32_t *tlbpc)
2789{
2790 uint32_t mps = UGETW(xfer->pipe->endpoint->edesc->wMaxPacketSize)(*(u_int16_t *)(xfer->pipe->endpoint->edesc->wMaxPacketSize
));
2791 uint32_t maxb, tdpc, residue, tbc;
2792
2793 /* Transfer Descriptor Packet Count, section 4.14.1. */
2794 tdpc = howmany(len, UE_GET_SIZE(mps))(((len) + ((((mps) & 0x7ff)) - 1)) / (((mps) & 0x7ff)
));
2795 if (tdpc == 0)
2796 tdpc = 1;
2797
2798 /* Transfer Burst Count */
2799 maxb = xhci_pipe_maxburst(xfer->pipe);
2800 tbc = howmany(tdpc, maxb + 1)(((tdpc) + ((maxb + 1) - 1)) / (maxb + 1)) - 1;
2801
2802 /* Transfer Last Burst Packet Count */
2803 if (xfer->device->speed == USB_SPEED_SUPER4) {
2804 residue = tdpc % (maxb + 1);
2805 if (residue == 0)
2806 *tlbpc = maxb;
2807 else
2808 *tlbpc = residue - 1;
2809 } else {
2810 *tlbpc = tdpc - 1;
2811 }
2812
2813 return (tbc);
2814}
2815
2816usbd_status
2817xhci_device_ctrl_transfer(struct usbd_xfer *xfer)
2818{
2819 usbd_status err;
2820
2821 err = usb_insert_transfer(xfer);
2822 if (err)
2823 return (err);
2824
2825 return (xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)((&xfer->pipe->queue)->sqh_first)));
2826}
2827
2828usbd_status
2829xhci_device_ctrl_start(struct usbd_xfer *xfer)
2830{
2831 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
2832 struct xhci_pipe *xp = (struct xhci_pipe *)xfer->pipe;
2833 struct xhci_trb *trb0, *trb;
2834 uint32_t flags, len = UGETW(xfer->request.wLength)(*(u_int16_t *)(xfer->request.wLength));
2835 uint8_t toggle;
2836 int s;
2837
2838 KASSERT(xfer->rqflags & URQ_REQUEST)((xfer->rqflags & 0x01) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/usb/xhci.c", 2838, "xfer->rqflags & URQ_REQUEST"
));
2839
2840 if (sc->sc_bus.dying || xp->halted)
2841 return (USBD_IOERROR);
2842
2843 if (xp->free_trbs < 3)
2844 return (USBD_NOMEM);
2845
2846 if (len != 0)
2847 usb_syncmem(&xfer->dmabuf, 0, len,
2848 usbd_xfer_isread(xfer) ?
2849 BUS_DMASYNC_PREREAD0x01 : BUS_DMASYNC_PREWRITE0x04);
2850
2851 /* We'll toggle the setup TRB once we're finished with the stages. */
2852 trb0 = xhci_xfer_get_trb(sc, xfer, &toggle, 0);
2853
2854 flags = XHCI_TRB_TYPE_SETUP(2 << 10) | XHCI_TRB_IDT(1 << 6) | (toggle ^ 1);
2855 if (len != 0) {
2856 if (usbd_xfer_isread(xfer))
2857 flags |= XHCI_TRB_TRT_IN(3 << 16);
2858 else
2859 flags |= XHCI_TRB_TRT_OUT(2 << 16);
2860 }
2861
2862 memcpy(&trb0->trb_paddr, &xfer->request, sizeof(trb0->trb_paddr))__builtin_memcpy((&trb0->trb_paddr), (&xfer->request
), (sizeof(trb0->trb_paddr)));
2863 trb0->trb_status = htole32(XHCI_TRB_INTR(0) | XHCI_TRB_LEN(8))((__uint32_t)((((0) & 0x3ff) << 22) | ((8) & 0x1ffff
)));
2864 trb0->trb_flags = htole32(flags)((__uint32_t)(flags));
2865 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2866 TRBOFF(&xp->ring, trb0), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2867 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
2868
2869 /* Data TRB */
2870 if (len != 0) {
2871 trb = xhci_xfer_get_trb(sc, xfer, &toggle, 0);
2872
2873 flags = XHCI_TRB_TYPE_DATA(3 << 10) | toggle;
2874 if (usbd_xfer_isread(xfer))
2875 flags |= XHCI_TRB_DIR_IN(1 << 16) | XHCI_TRB_ISP(1 << 2);
2876
2877 trb->trb_paddr = htole64(DMAADDR(&xfer->dmabuf, 0))((__uint64_t)(((&xfer->dmabuf)->block->map->dm_segs
[0].ds_addr + (&xfer->dmabuf)->offs + (0))));
2878 trb->trb_status = htole32(((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, len, len)))
2879 XHCI_TRB_INTR(0) | XHCI_TRB_LEN(len) |((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, len, len)))
2880 xhci_xfer_tdsize(xfer, len, len)((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, len, len)))
2881 )((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, len, len)));
2882 trb->trb_flags = htole32(flags)((__uint32_t)(flags));
2883
2884 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2885 TRBOFF(&xp->ring, trb), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2886 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
2887 }
2888
2889 /* Status TRB */
2890 trb = xhci_xfer_get_trb(sc, xfer, &toggle, 1);
2891
2892 flags = XHCI_TRB_TYPE_STATUS(4 << 10) | XHCI_TRB_IOC(1 << 5) | toggle;
2893 if (len == 0 || !usbd_xfer_isread(xfer))
2894 flags |= XHCI_TRB_DIR_IN(1 << 16);
2895
2896 trb->trb_paddr = 0;
2897 trb->trb_status = htole32(XHCI_TRB_INTR(0))((__uint32_t)((((0) & 0x3ff) << 22)));
2898 trb->trb_flags = htole32(flags)((__uint32_t)(flags));
2899
2900 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2901 TRBOFF(&xp->ring, trb), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2902 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
2903
2904 /* Setup TRB */
2905 trb0->trb_flags ^= htole32(XHCI_TRB_CYCLE)((__uint32_t)((1 << 0)));
2906 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2907 TRBOFF(&xp->ring, trb0), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2908 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
2909
2910 s = splusb()splraise(0x2);
2911 XDWRITE4(sc, XHCI_DOORBELL(xp->slot), xp->dci)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_door_off
+ ((0x0000 + (4 * (xp->slot))))), ((xp->dci))));
2912
2913 xfer->status = USBD_IN_PROGRESS;
2914 if (xfer->timeout && !sc->sc_bus.use_polling) {
2915 timeout_del(&xfer->timeout_handle);
2916 timeout_set(&xfer->timeout_handle, xhci_timeout, xfer);
2917 timeout_add_msec(&xfer->timeout_handle, xfer->timeout);
2918 }
2919 splx(s)spllower(s);
2920
2921 return (USBD_IN_PROGRESS);
2922}
2923
2924void
2925xhci_device_ctrl_abort(struct usbd_xfer *xfer)
2926{
2927 xhci_abort_xfer(xfer, USBD_CANCELLED);
2928}
2929
2930usbd_status
2931xhci_device_generic_transfer(struct usbd_xfer *xfer)
2932{
2933 usbd_status err;
2934
2935 err = usb_insert_transfer(xfer);
2936 if (err)
2937 return (err);
2938
2939 return (xhci_device_generic_start(SIMPLEQ_FIRST(&xfer->pipe->queue)((&xfer->pipe->queue)->sqh_first)));
2940}
2941
2942usbd_status
2943xhci_device_generic_start(struct usbd_xfer *xfer)
2944{
2945 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
2946 struct xhci_pipe *xp = (struct xhci_pipe *)xfer->pipe;
2947 struct xhci_trb *trb0, *trb;
2948 uint32_t len, remain, flags;
2949 uint32_t mps = UGETW(xfer->pipe->endpoint->edesc->wMaxPacketSize)(*(u_int16_t *)(xfer->pipe->endpoint->edesc->wMaxPacketSize
));
2950 uint64_t paddr = DMAADDR(&xfer->dmabuf, 0)((&xfer->dmabuf)->block->map->dm_segs[0].ds_addr
+ (&xfer->dmabuf)->offs + (0));
2951 uint8_t toggle;
2952 int s, i, ntrb, zerotd = 0;
2953
2954 KASSERT(!(xfer->rqflags & URQ_REQUEST))((!(xfer->rqflags & 0x01)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/usb/xhci.c", 2954, "!(xfer->rqflags & URQ_REQUEST)"
));
2955
2956 if (sc->sc_bus.dying || xp->halted)
2957 return (USBD_IOERROR);
2958
2959 /* How many TRBs do we need for this transfer? */
2960 ntrb = howmany(xfer->length, XHCI_TRB_MAXSIZE)(((xfer->length) + (((64 * 1024)) - 1)) / ((64 * 1024)));
2961
2962 /* If the buffer crosses a 64k boundary, we need one more. */
2963 len = XHCI_TRB_MAXSIZE(64 * 1024) - (paddr & (XHCI_TRB_MAXSIZE(64 * 1024) - 1));
2964 if (len < xfer->length)
2965 ntrb = howmany(xfer->length - len, XHCI_TRB_MAXSIZE)(((xfer->length - len) + (((64 * 1024)) - 1)) / ((64 * 1024
))) + 1;
2966 else
2967 len = xfer->length;
2968
2969 /* If we need to append a zero length packet, we need one more. */
2970 if ((xfer->flags & USBD_FORCE_SHORT_XFER0x08 || xfer->length == 0) &&
2971 (xfer->length % UE_GET_SIZE(mps)((mps) & 0x7ff) == 0))
2972 zerotd = 1;
2973
2974 if (xp->free_trbs < (ntrb + zerotd))
2975 return (USBD_NOMEM);
2976
2977 usb_syncmem(&xfer->dmabuf, 0, xfer->length,
2978 usbd_xfer_isread(xfer) ?
2979 BUS_DMASYNC_PREREAD0x01 : BUS_DMASYNC_PREWRITE0x04);
2980
2981 /* We'll toggle the first TRB once we're finished with the chain. */
2982 trb0 = xhci_xfer_get_trb(sc, xfer, &toggle, (ntrb == 1));
2983 flags = XHCI_TRB_TYPE_NORMAL(1 << 10) | (toggle ^ 1);
2984 if (usbd_xfer_isread(xfer))
2985 flags |= XHCI_TRB_ISP(1 << 2);
2986 flags |= (ntrb == 1) ? XHCI_TRB_IOC(1 << 5) : XHCI_TRB_CHAIN(1 << 4);
2987
2988 trb0->trb_paddr = htole64(DMAADDR(&xfer->dmabuf, 0))((__uint64_t)(((&xfer->dmabuf)->block->map->dm_segs
[0].ds_addr + (&xfer->dmabuf)->offs + (0))));
2989 trb0->trb_status = htole32(((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->length, len)))
2990 XHCI_TRB_INTR(0) | XHCI_TRB_LEN(len) |((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->length, len)))
2991 xhci_xfer_tdsize(xfer, xfer->length, len)((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->length, len)))
2992 )((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->length, len)));
2993 trb0->trb_flags = htole32(flags)((__uint32_t)(flags));
2994 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2995 TRBOFF(&xp->ring, trb0), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
2996 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
2997
2998 remain = xfer->length - len;
2999 paddr += len;
3000
3001 /* Chain more TRBs if needed. */
3002 for (i = ntrb - 1; i > 0; i--) {
3003 len = min(remain, XHCI_TRB_MAXSIZE(64 * 1024));
3004
3005 /* Next (or Last) TRB. */
3006 trb = xhci_xfer_get_trb(sc, xfer, &toggle, (i == 1));
3007 flags = XHCI_TRB_TYPE_NORMAL(1 << 10) | toggle;
3008 if (usbd_xfer_isread(xfer))
3009 flags |= XHCI_TRB_ISP(1 << 2);
3010 flags |= (i == 1) ? XHCI_TRB_IOC(1 << 5) : XHCI_TRB_CHAIN(1 << 4);
3011
3012 trb->trb_paddr = htole64(paddr)((__uint64_t)(paddr));
3013 trb->trb_status = htole32(((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)))
3014 XHCI_TRB_INTR(0) | XHCI_TRB_LEN(len) |((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)))
3015 xhci_xfer_tdsize(xfer, remain, len)((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)))
3016 )((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)));
3017 trb->trb_flags = htole32(flags)((__uint32_t)(flags));
3018
3019 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3020 TRBOFF(&xp->ring, trb), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3021 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
3022
3023 remain -= len;
3024 paddr += len;
3025 }
3026
3027 /* Do we need to issue a zero length transfer? */
3028 if (zerotd == 1) {
3029 trb = xhci_xfer_get_trb(sc, xfer, &toggle, -1);
3030 trb->trb_paddr = 0;
3031 trb->trb_status = 0;
3032 trb->trb_flags = htole32(XHCI_TRB_TYPE_NORMAL | XHCI_TRB_IOC | toggle)((__uint32_t)((1 << 10) | (1 << 5) | toggle));
3033 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3034 TRBOFF(&xp->ring, trb), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3035 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
3036 }
3037
3038 /* First TRB. */
3039 trb0->trb_flags ^= htole32(XHCI_TRB_CYCLE)((__uint32_t)((1 << 0)));
3040 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3041 TRBOFF(&xp->ring, trb0), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3042 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
3043
3044 s = splusb()splraise(0x2);
3045 XDWRITE4(sc, XHCI_DOORBELL(xp->slot), xp->dci)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_door_off
+ ((0x0000 + (4 * (xp->slot))))), ((xp->dci))));
3046
3047 xfer->status = USBD_IN_PROGRESS;
3048 if (xfer->timeout && !sc->sc_bus.use_polling) {
3049 timeout_del(&xfer->timeout_handle);
3050 timeout_set(&xfer->timeout_handle, xhci_timeout, xfer);
3051 timeout_add_msec(&xfer->timeout_handle, xfer->timeout);
3052 }
3053 splx(s)spllower(s);
3054
3055 return (USBD_IN_PROGRESS);
3056}
3057
3058void
3059xhci_device_generic_done(struct usbd_xfer *xfer)
3060{
3061 /* Only happens with interrupt transfers. */
3062 if (xfer->pipe->repeat) {
3063 xfer->actlen = 0;
3064 xhci_device_generic_start(xfer);
3065 }
3066}
3067
3068void
3069xhci_device_generic_abort(struct usbd_xfer *xfer)
3070{
3071 KASSERT(!xfer->pipe->repeat || xfer->pipe->intrxfer == xfer)((!xfer->pipe->repeat || xfer->pipe->intrxfer == xfer
) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c"
, 3071, "!xfer->pipe->repeat || xfer->pipe->intrxfer == xfer"
));
3072
3073 xhci_abort_xfer(xfer, USBD_CANCELLED);
3074}
3075
3076usbd_status
3077xhci_device_isoc_transfer(struct usbd_xfer *xfer)
3078{
3079 usbd_status err;
3080
3081 err = usb_insert_transfer(xfer);
3082 if (err && err != USBD_IN_PROGRESS)
3083 return (err);
3084
3085 return (xhci_device_isoc_start(xfer));
3086}
3087
3088usbd_status
3089xhci_device_isoc_start(struct usbd_xfer *xfer)
3090{
3091 struct xhci_softc *sc = (struct xhci_softc *)xfer->device->bus;
3092 struct xhci_pipe *xp = (struct xhci_pipe *)xfer->pipe;
3093 struct xhci_xfer *xx = (struct xhci_xfer *)xfer;
3094 struct xhci_trb *trb0, *trb;
3095 uint32_t len, remain, flags;
3096 uint64_t paddr;
3097 uint32_t tbc, tlbpc;
3098 int s, i, j, ntrb = xfer->nframes;
3099 uint8_t toggle;
3100
3101 KASSERT(!(xfer->rqflags & URQ_REQUEST))((!(xfer->rqflags & 0x01)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/usb/xhci.c", 3101, "!(xfer->rqflags & URQ_REQUEST)"
));
3102
3103 /*
3104 * To allow continuous transfers, above we start all transfers
3105 * immediately. However, we're still going to get usbd_start_next call
3106 * this when another xfer completes. So, check if this is already
3107 * in progress or not
3108 */
3109 if (xx->ntrb > 0)
3110 return (USBD_IN_PROGRESS);
3111
3112 if (sc->sc_bus.dying || xp->halted)
3113 return (USBD_IOERROR);
3114
3115 /* Why would you do that anyway? */
3116 if (sc->sc_bus.use_polling)
3117 return (USBD_INVAL);
3118
3119 paddr = DMAADDR(&xfer->dmabuf, 0)((&xfer->dmabuf)->block->map->dm_segs[0].ds_addr
+ (&xfer->dmabuf)->offs + (0));
3120
3121 /* How many TRBs do for all Transfers? */
3122 for (i = 0, ntrb = 0; i < xfer->nframes; i++) {
3123 /* How many TRBs do we need for this transfer? */
3124 ntrb += howmany(xfer->frlengths[i], XHCI_TRB_MAXSIZE)(((xfer->frlengths[i]) + (((64 * 1024)) - 1)) / ((64 * 1024
)));
3125
3126 /* If the buffer crosses a 64k boundary, we need one more. */
3127 len = XHCI_TRB_MAXSIZE(64 * 1024) - (paddr & (XHCI_TRB_MAXSIZE(64 * 1024) - 1));
3128 if (len < xfer->frlengths[i])
3129 ntrb++;
3130
3131 paddr += xfer->frlengths[i];
3132 }
3133
3134 if (xp->free_trbs < ntrb)
3135 return (USBD_NOMEM);
3136
3137 usb_syncmem(&xfer->dmabuf, 0, xfer->length,
3138 usbd_xfer_isread(xfer) ?
3139 BUS_DMASYNC_PREREAD0x01 : BUS_DMASYNC_PREWRITE0x04);
3140
3141 paddr = DMAADDR(&xfer->dmabuf, 0)((&xfer->dmabuf)->block->map->dm_segs[0].ds_addr
+ (&xfer->dmabuf)->offs + (0));
3142
3143 for (i = 0, trb0 = NULL((void *)0); i < xfer->nframes; i++) {
3144 /* How many TRBs do we need for this transfer? */
3145 ntrb = howmany(xfer->frlengths[i], XHCI_TRB_MAXSIZE)(((xfer->frlengths[i]) + (((64 * 1024)) - 1)) / ((64 * 1024
)));
3146
3147 /* If the buffer crosses a 64k boundary, we need one more. */
3148 len = XHCI_TRB_MAXSIZE(64 * 1024) - (paddr & (XHCI_TRB_MAXSIZE(64 * 1024) - 1));
3149 if (len < xfer->frlengths[i])
3150 ntrb++;
3151 else
3152 len = xfer->frlengths[i];
3153
3154 KASSERT(ntrb < 3)((ntrb < 3) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/usb/xhci.c"
, 3154, "ntrb < 3"));
3155
3156 /*
3157 * We'll commit the first TRB once we're finished with the
3158 * chain.
3159 */
3160 trb = xhci_xfer_get_trb(sc, xfer, &toggle, (ntrb == 1));
3161
3162 DPRINTFN(4, ("%s:%d: ring %p trb0_idx %lu ntrb %d paddr %llx "
3163 "len %u\n", __func__, __LINE__,
3164 &xp->ring.trbs[0], (trb - &xp->ring.trbs[0]), ntrb, paddr,
3165 len));
3166
3167 /* Record the first TRB so we can toggle later. */
3168 if (trb0 == NULL((void *)0)) {
3169 trb0 = trb;
3170 toggle ^= 1;
3171 }
3172
3173 flags = XHCI_TRB_TYPE_ISOCH(5 << 10) | XHCI_TRB_SIA(1U << 31) | toggle;
3174 if (usbd_xfer_isread(xfer))
3175 flags |= XHCI_TRB_ISP(1 << 2);
3176 flags |= (ntrb == 1) ? XHCI_TRB_IOC(1 << 5) : XHCI_TRB_CHAIN(1 << 4);
3177
3178 tbc = xhci_xfer_tbc(xfer, xfer->frlengths[i], &tlbpc);
3179 flags |= XHCI_TRB_ISOC_TBC(tbc)(((tbc) & 0x3) << 7) | XHCI_TRB_ISOC_TLBPC(tlbpc)(((tlbpc) & 0xf) << 16);
3180
3181 trb->trb_paddr = htole64(paddr)((__uint64_t)(paddr));
3182 trb->trb_status = htole32(((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->frlengths[i], len)))
3183 XHCI_TRB_INTR(0) | XHCI_TRB_LEN(len) |((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->frlengths[i], len)))
3184 xhci_xfer_tdsize(xfer, xfer->frlengths[i], len)((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->frlengths[i], len)))
3185 )((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, xfer->frlengths[i], len)));
3186 trb->trb_flags = htole32(flags)((__uint32_t)(flags));
3187
3188 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3189 TRBOFF(&xp->ring, trb), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3190 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
3191
3192 remain = xfer->frlengths[i] - len;
3193 paddr += len;
3194
3195 /* Chain more TRBs if needed. */
3196 for (j = ntrb - 1; j > 0; j--) {
3197 len = min(remain, XHCI_TRB_MAXSIZE(64 * 1024));
3198
3199 /* Next (or Last) TRB. */
3200 trb = xhci_xfer_get_trb(sc, xfer, &toggle, (j == 1));
3201 flags = XHCI_TRB_TYPE_NORMAL(1 << 10) | toggle;
3202 if (usbd_xfer_isread(xfer))
3203 flags |= XHCI_TRB_ISP(1 << 2);
3204 flags |= (j == 1) ? XHCI_TRB_IOC(1 << 5) : XHCI_TRB_CHAIN(1 << 4);
3205 DPRINTFN(3, ("%s:%d: ring %p trb0_idx %lu ntrb %d "
3206 "paddr %llx len %u\n", __func__, __LINE__,
3207 &xp->ring.trbs[0], (trb - &xp->ring.trbs[0]), ntrb,
3208 paddr, len));
3209
3210 trb->trb_paddr = htole64(paddr)((__uint64_t)(paddr));
3211 trb->trb_status = htole32(((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)))
3212 XHCI_TRB_INTR(0) | XHCI_TRB_LEN(len) |((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)))
3213 xhci_xfer_tdsize(xfer, remain, len)((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)))
3214 )((__uint32_t)((((0) & 0x3ff) << 22) | ((len) & 0x1ffff
) | xhci_xfer_tdsize(xfer, remain, len)));
3215 trb->trb_flags = htole32(flags)((__uint32_t)(flags));
3216
3217 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3218 TRBOFF(&xp->ring, trb), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3219 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
3220
3221 remain -= len;
3222 paddr += len;
3223 }
3224
3225 xfer->frlengths[i] = 0;
3226 }
3227
3228 /* First TRB. */
3229 trb0->trb_flags ^= htole32(XHCI_TRB_CYCLE)((__uint32_t)((1 << 0)));
3230 bus_dmamap_sync(xp->ring.dma.tag, xp->ring.dma.map,(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3231 TRBOFF(&xp->ring, trb0), sizeof(struct xhci_trb),(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04))
3232 BUS_DMASYNC_PREWRITE)(*(xp->ring.dma.tag)->_dmamap_sync)((xp->ring.dma.tag
), (xp->ring.dma.map), (((char *)(trb0) - (char *)((&xp
->ring)->trbs))), (sizeof(struct xhci_trb)), (0x04));
3233
3234 s = splusb()splraise(0x2);
3235 XDWRITE4(sc, XHCI_DOORBELL(xp->slot), xp->dci)(((sc)->iot)->write_4(((sc)->ioh), ((sc)->sc_door_off
+ ((0x0000 + (4 * (xp->slot))))), ((xp->dci))));
3236
3237 xfer->status = USBD_IN_PROGRESS;
3238
3239 if (xfer->timeout) {
3240 timeout_del(&xfer->timeout_handle);
3241 timeout_set(&xfer->timeout_handle, xhci_timeout, xfer);
3242 timeout_add_msec(&xfer->timeout_handle, xfer->timeout);
3243 }
3244 splx(s)spllower(s);
3245
3246 return (USBD_IN_PROGRESS);
3247}