Bug Summary

File:dev/acpi/acpidmar.c
Warning:line 967, column 9
The left expression of the compound assignment is an uninitialized value. The computed value will also be garbage

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name acpidmar.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/acpi/acpidmar.c
1/*
2 * Copyright (c) 2015 Jordan Hargrave <jordan_hargrave@hotmail.com>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16
17#include <sys/param.h>
18#include <sys/systm.h>
19#include <sys/kernel.h>
20#include <sys/device.h>
21#include <sys/malloc.h>
22#include <sys/queue.h>
23#include <sys/types.h>
24#include <sys/mbuf.h>
25#include <sys/proc.h>
26
27#include <uvm/uvm_extern.h>
28
29#include <machine/apicvar.h>
30#include <machine/biosvar.h>
31#include <machine/cpuvar.h>
32#include <machine/bus.h>
33
34#include <dev/acpi/acpireg.h>
35#include <dev/acpi/acpivar.h>
36#include <dev/acpi/acpidev.h>
37#include <dev/acpi/amltypes.h>
38#include <dev/acpi/dsdt.h>
39
40#include <uvm/uvm_extern.h>
41
42#include <machine/i8259.h>
43#include <machine/i82093reg.h>
44#include <machine/i82093var.h>
45#include <machine/i82489reg.h>
46#include <machine/i82489var.h>
47
48#include <machine/mpbiosvar.h>
49
50#include <dev/pci/pcireg.h>
51#include <dev/pci/pcivar.h>
52#include <dev/pci/pcidevs.h>
53#include <dev/pci/ppbreg.h>
54
55#include "ioapic.h"
56
57#include "acpidmar.h"
58#include "amd_iommu.h"
59
60/* We don't want IOMMU to remap MSI */
61#define MSI_BASE_ADDRESS0xFEE00000L 0xFEE00000L
62#define MSI_BASE_SIZE0x00100000L 0x00100000L
63#define MAX_DEVFN65536 65536
64
65#ifdef IOMMU_DEBUG
66int acpidmar_dbg_lvl = 0;
67#define DPRINTF(lvl,x...) if (acpidmar_dbg_lvl >= lvl) { printf(x); }
68#else
69#define DPRINTF(lvl,x...)
70#endif
71
72#ifdef DDB1
73int acpidmar_ddb = 0;
74#endif
75
76int acpidmar_force_cm = 1;
77
78/* Page Table Entry per domain */
79struct iommu_softc;
80
81static inline int
82mksid(int b, int d, int f)
83{
84 return (b << 8) + (d << 3) + f;
85}
86
87static inline int
88sid_devfn(int sid)
89{
90 return sid & 0xff;
91}
92
93static inline int
94sid_bus(int sid)
95{
96 return (sid >> 8) & 0xff;
97}
98
99static inline int
100sid_dev(int sid)
101{
102 return (sid >> 3) & 0x1f;
103}
104
105static inline int
106sid_fun(int sid)
107{
108 return (sid >> 0) & 0x7;
109}
110
111/* Alias mapping */
112#define SID_INVALID0x80000000L 0x80000000L
113static uint32_t sid_flag[MAX_DEVFN65536];
114
115struct domain_dev {
116 int sid;
117 int sec;
118 int sub;
119 TAILQ_ENTRY(domain_dev)struct { struct domain_dev *tqe_next; struct domain_dev **tqe_prev
; } link;
120};
121
122struct domain {
123 struct iommu_softc *iommu;
124 int did;
125 int gaw;
126 struct pte_entry *pte;
127 paddr_t ptep;
128 struct bus_dma_tag dmat;
129 int flag;
130
131 struct mutex exlck;
132 char exname[32];
133 struct extent *iovamap;
134 TAILQ_HEAD(,domain_dev)struct { struct domain_dev *tqh_first; struct domain_dev **tqh_last
; } devices;
135 TAILQ_ENTRY(domain)struct { struct domain *tqe_next; struct domain **tqe_prev; } link;
136};
137
138#define DOM_DEBUG0x1 0x1
139#define DOM_NOMAP0x2 0x2
140
141struct dmar_devlist {
142 int type;
143 int bus;
144 int ndp;
145 struct acpidmar_devpath *dp;
146 TAILQ_ENTRY(dmar_devlist)struct { struct dmar_devlist *tqe_next; struct dmar_devlist *
*tqe_prev; } link;
147};
148
149TAILQ_HEAD(devlist_head, dmar_devlist)struct devlist_head { struct dmar_devlist *tqh_first; struct dmar_devlist
**tqh_last; };
150
151struct ivhd_devlist {
152 int start_id;
153 int end_id;
154 int cfg;
155 TAILQ_ENTRY(ivhd_devlist)struct { struct ivhd_devlist *tqe_next; struct ivhd_devlist *
*tqe_prev; } link;
156};
157
158struct rmrr_softc {
159 TAILQ_ENTRY(rmrr_softc)struct { struct rmrr_softc *tqe_next; struct rmrr_softc **tqe_prev
; } link;
160 struct devlist_head devices;
161 int segment;
162 uint64_t start;
163 uint64_t end;
164};
165
166struct atsr_softc {
167 TAILQ_ENTRY(atsr_softc)struct { struct atsr_softc *tqe_next; struct atsr_softc **tqe_prev
; } link;
168 struct devlist_head devices;
169 int segment;
170 int flags;
171};
172
173struct iommu_pic {
174 struct pic pic;
175 struct iommu_softc *iommu;
176};
177
178#define IOMMU_FLAGS_CATCHALL0x1 0x1
179#define IOMMU_FLAGS_BAD0x2 0x2
180#define IOMMU_FLAGS_SUSPEND0x4 0x4
181
182struct iommu_softc {
183 TAILQ_ENTRY(iommu_softc)struct { struct iommu_softc *tqe_next; struct iommu_softc **tqe_prev
; }link;
184 struct devlist_head devices;
185 int id;
186 int flags;
187 int segment;
188
189 struct mutex reg_lock;
190
191 bus_space_tag_t iot;
192 bus_space_handle_t ioh;
193
194 uint64_t cap;
195 uint64_t ecap;
196 uint32_t gcmd;
197
198 int mgaw;
199 int agaw;
200 int ndoms;
201
202 struct root_entry *root;
203 struct context_entry *ctx[256];
204
205 void *intr;
206 struct iommu_pic pic;
207 int fedata;
208 uint64_t feaddr;
209 uint64_t rtaddr;
210
211 /* Queued Invalidation */
212 int qi_head;
213 int qi_tail;
214 paddr_t qip;
215 struct qi_entry *qi;
216
217 struct domain *unity;
218 TAILQ_HEAD(,domain)struct { struct domain *tqh_first; struct domain **tqh_last; } domains;
219
220 /* AMD iommu */
221 struct ivhd_dte *dte;
222 void *cmd_tbl;
223 void *evt_tbl;
224 paddr_t cmd_tblp;
225 paddr_t evt_tblp;
226};
227
228static inline int iommu_bad(struct iommu_softc *sc)
229{
230 return (sc->flags & IOMMU_FLAGS_BAD0x2);
231}
232
233static inline int iommu_enabled(struct iommu_softc *sc)
234{
235 if (sc->dte) {
236 return 1;
237 }
238 return (sc->gcmd & GCMD_TE(1LL << 31));
239}
240
241struct acpidmar_softc {
242 struct device sc_dev;
243
244 pci_chipset_tag_t sc_pc;
245 bus_space_tag_t sc_memt;
246 int sc_haw;
247 int sc_flags;
248 bus_dma_tag_t sc_dmat;
249
250 struct ivhd_dte *sc_hwdte;
251 paddr_t sc_hwdtep;
252
253 TAILQ_HEAD(,iommu_softc)struct { struct iommu_softc *tqh_first; struct iommu_softc **
tqh_last; }sc_drhds;
254 TAILQ_HEAD(,rmrr_softc)struct { struct rmrr_softc *tqh_first; struct rmrr_softc **tqh_last
; } sc_rmrrs;
255 TAILQ_HEAD(,atsr_softc)struct { struct atsr_softc *tqh_first; struct atsr_softc **tqh_last
; } sc_atsrs;
256};
257
258int acpidmar_activate(struct device *, int);
259int acpidmar_match(struct device *, void *, void *);
260void acpidmar_attach(struct device *, struct device *, void *);
261struct domain *acpidmar_pci_attach(struct acpidmar_softc *, int, int, int);
262
263struct cfattach acpidmar_ca = {
264 sizeof(struct acpidmar_softc), acpidmar_match, acpidmar_attach, NULL((void *)0),
265 acpidmar_activate
266};
267
268struct cfdriver acpidmar_cd = {
269 NULL((void *)0), "acpidmar", DV_DULL
270};
271
272struct acpidmar_softc *acpidmar_sc;
273int acpidmar_intr(void *);
274int acpiivhd_intr(void *);
275
276#define DID_UNITY0x1 0x1
277
278void _dumppte(struct pte_entry *, int, vaddr_t);
279
280struct domain *domain_create(struct iommu_softc *, int);
281struct domain *domain_lookup(struct acpidmar_softc *, int, int);
282
283void domain_unload_map(struct domain *, bus_dmamap_t);
284void domain_load_map(struct domain *, bus_dmamap_t, int, int, const char *);
285
286void (*domain_map_page)(struct domain *, vaddr_t, paddr_t, uint64_t);
287void domain_map_page_amd(struct domain *, vaddr_t, paddr_t, uint64_t);
288void domain_map_page_intel(struct domain *, vaddr_t, paddr_t, uint64_t);
289void domain_map_pthru(struct domain *, paddr_t, paddr_t);
290
291void acpidmar_pci_hook(pci_chipset_tag_t, struct pci_attach_args *);
292void acpidmar_parse_devscope(union acpidmar_entry *, int, int,
293 struct devlist_head *);
294int acpidmar_match_devscope(struct devlist_head *, pci_chipset_tag_t, int);
295
296void acpidmar_init(struct acpidmar_softc *, struct acpi_dmar *);
297void acpidmar_drhd(struct acpidmar_softc *, union acpidmar_entry *);
298void acpidmar_rmrr(struct acpidmar_softc *, union acpidmar_entry *);
299void acpidmar_atsr(struct acpidmar_softc *, union acpidmar_entry *);
300void acpiivrs_init(struct acpidmar_softc *, struct acpi_ivrs *);
301
302void *acpidmar_intr_establish(void *, int, int (*)(void *), void *,
303 const char *);
304
305void iommu_write_4(struct iommu_softc *, int, uint32_t);
306uint32_t iommu_read_4(struct iommu_softc *, int);
307void iommu_write_8(struct iommu_softc *, int, uint64_t);
308uint64_t iommu_read_8(struct iommu_softc *, int);
309void iommu_showfault(struct iommu_softc *, int,
310 struct fault_entry *);
311void iommu_showcfg(struct iommu_softc *, int);
312
313int iommu_init(struct acpidmar_softc *, struct iommu_softc *,
314 struct acpidmar_drhd *);
315int iommu_enable_translation(struct iommu_softc *, int);
316void iommu_enable_qi(struct iommu_softc *, int);
317void iommu_flush_cache(struct iommu_softc *, void *, size_t);
318void *iommu_alloc_page(struct iommu_softc *, paddr_t *);
319void iommu_flush_write_buffer(struct iommu_softc *);
320void iommu_issue_qi(struct iommu_softc *, struct qi_entry *);
321
322void iommu_flush_ctx(struct iommu_softc *, int, int, int, int);
323void iommu_flush_ctx_qi(struct iommu_softc *, int, int, int, int);
324void iommu_flush_tlb(struct iommu_softc *, int, int);
325void iommu_flush_tlb_qi(struct iommu_softc *, int, int);
326
327void iommu_set_rtaddr(struct iommu_softc *, paddr_t);
328
329void *iommu_alloc_hwdte(struct acpidmar_softc *, size_t, paddr_t *);
330
331const char *dmar_bdf(int);
332
333const char *
334dmar_bdf(int sid)
335{
336 static char bdf[32];
337
338 snprintf(bdf, sizeof(bdf), "%.4x:%.2x:%.2x.%x", 0,
339 sid_bus(sid), sid_dev(sid), sid_fun(sid));
340
341 return (bdf);
342}
343
344/* busdma */
345static int dmar_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
346 bus_size_t, int, bus_dmamap_t *);
347static void dmar_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
348static int dmar_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
349 struct proc *, int);
350static int dmar_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *,
351 int);
352static int dmar_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t, struct uio *, int);
353static int dmar_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t,
354 bus_dma_segment_t *, int, bus_size_t, int);
355static void dmar_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
356static void dmar_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
357 bus_size_t, int);
358static int dmar_dmamem_alloc(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
359 bus_dma_segment_t *, int, int *, int);
360static void dmar_dmamem_free(bus_dma_tag_t, bus_dma_segment_t *, int);
361static int dmar_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *, int, size_t,
362 caddr_t *, int);
363static void dmar_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
364static paddr_t dmar_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, int, off_t,
365 int, int);
366
367static void dmar_dumpseg(bus_dma_tag_t, int, bus_dma_segment_t *, const char *);
368const char *dom_bdf(struct domain *);
369void domain_map_check(struct domain *);
370
371struct pte_entry *pte_lvl(struct iommu_softc *, struct pte_entry *, vaddr_t, int, uint64_t);
372int ivhd_poll_events(struct iommu_softc *);
373void ivhd_showreg(struct iommu_softc *);
374void ivhd_showdte(struct iommu_softc *);
375void ivhd_showcmd(struct iommu_softc *);
376
377static inline int
378debugme(struct domain *dom)
379{
380 return 0;
381 return (dom->flag & DOM_DEBUG0x1);
382}
383
384void
385domain_map_check(struct domain *dom)
386{
387 struct iommu_softc *iommu;
388 struct domain_dev *dd;
389 struct context_entry *ctx;
390 int v;
391
392 iommu = dom->iommu;
393 TAILQ_FOREACH(dd, &dom->devices, link)for((dd) = ((&dom->devices)->tqh_first); (dd) != ((
void *)0); (dd) = ((dd)->link.tqe_next)) {
394 acpidmar_pci_attach(acpidmar_sc, iommu->segment, dd->sid, 1);
395
396 if (iommu->dte)
397 continue;
398
399 /* Check if this is the first time we are mapped */
400 ctx = &iommu->ctx[sid_bus(dd->sid)][sid_devfn(dd->sid)];
401 v = context_user(ctx);
402 if (v != 0xA) {
403 printf(" map: %.4x:%.2x:%.2x.%x iommu:%d did:%.4x\n",
404 iommu->segment,
405 sid_bus(dd->sid),
406 sid_dev(dd->sid),
407 sid_fun(dd->sid),
408 iommu->id,
409 dom->did);
410 context_set_user(ctx, 0xA);
411 }
412 }
413}
414
415/* Map a single page as passthrough - used for DRM */
416void
417dmar_ptmap(bus_dma_tag_t tag, bus_addr_t addr)
418{
419 struct domain *dom = tag->_cookie;
420
421 if (!acpidmar_sc)
422 return;
423 domain_map_check(dom);
424 domain_map_page(dom, addr, addr, PTE_P(1L << 0) | PTE_R0x00 | PTE_W(1L << 1));
425}
426
427/* Map a range of pages 1:1 */
428void
429domain_map_pthru(struct domain *dom, paddr_t start, paddr_t end)
430{
431 domain_map_check(dom);
432 while (start < end) {
433 domain_map_page(dom, start, start, PTE_P(1L << 0) | PTE_R0x00 | PTE_W(1L << 1));
434 start += VTD_PAGE_SIZE4096;
435 }
436}
437
438/* Map a single paddr to IOMMU paddr */
439void
440domain_map_page_intel(struct domain *dom, vaddr_t va, paddr_t pa, uint64_t flags)
441{
442 paddr_t paddr;
443 struct pte_entry *pte, *npte;
444 int lvl, idx;
445 struct iommu_softc *iommu;
446
447 iommu = dom->iommu;
448 /* Insert physical address into virtual address map
449 * XXX: could we use private pmap here?
450 * essentially doing a pmap_enter(map, va, pa, prot);
451 */
452
453 /* Only handle 4k pages for now */
454 npte = dom->pte;
455 for (lvl = iommu->agaw - VTD_STRIDE_SIZE9; lvl>= VTD_LEVEL012;
456 lvl -= VTD_STRIDE_SIZE9) {
457 idx = (va >> lvl) & VTD_STRIDE_MASK0x1FF;
458 pte = &npte[idx];
459 if (lvl == VTD_LEVEL012) {
460 /* Level 1: Page Table - add physical address */
461 pte->val = pa | flags;
462 iommu_flush_cache(iommu, pte, sizeof(*pte));
463 break;
464 } else if (!(pte->val & PTE_P(1L << 0))) {
465 /* Level N: Point to lower level table */
466 iommu_alloc_page(iommu, &paddr);
467 pte->val = paddr | PTE_P(1L << 0) | PTE_R0x00 | PTE_W(1L << 1);
468 iommu_flush_cache(iommu, pte, sizeof(*pte));
469 }
470 npte = (void *)PMAP_DIRECT_MAP((pte->val & VTD_PTE_MASK))((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + ((pte->val & 0x0000FFFFFFFFF000LL)));
471 }
472}
473
474/* Map a single paddr to IOMMU paddr: AMD
475 * physical address breakdown into levels:
476 * xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
477 * 5.55555555.44444444.43333333,33222222.22211111.1111----.--------
478 * mode:
479 * 000 = none shift
480 * 001 = 1 [21].12
481 * 010 = 2 [30].21
482 * 011 = 3 [39].30
483 * 100 = 4 [48].39
484 * 101 = 5 [57]
485 * 110 = 6
486 * 111 = reserved
487 */
488struct pte_entry *
489pte_lvl(struct iommu_softc *iommu, struct pte_entry *pte, vaddr_t va,
490 int shift, uint64_t flags)
491{
492 paddr_t paddr;
493 int idx;
494
495 idx = (va >> shift) & VTD_STRIDE_MASK0x1FF;
496 if (!(pte[idx].val & PTE_P(1L << 0))) {
497 /* Page Table entry is not present... create a new page entry */
498 iommu_alloc_page(iommu, &paddr);
499 pte[idx].val = paddr | flags;
500 iommu_flush_cache(iommu, &pte[idx], sizeof(pte[idx]));
501 }
502 return (void *)PMAP_DIRECT_MAP((pte[idx].val & PTE_PADDR_MASK))((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + ((pte[idx].val & 0x000FFFFFFFFFF000LL)));
503}
504
505void
506domain_map_page_amd(struct domain *dom, vaddr_t va, paddr_t pa, uint64_t flags)
507{
508 struct pte_entry *pte;
509 struct iommu_softc *iommu;
510 int idx;
511
512 iommu = dom->iommu;
513 /* Insert physical address into virtual address map
514 * XXX: could we use private pmap here?
515 * essentially doing a pmap_enter(map, va, pa, prot);
516 */
517
518 /* Always assume AMD levels=4 */
519 /* 39 30 21 12 */
520 /* ---------|---------|---------|---------|------------ */
521 pte = dom->pte;
522 pte = pte_lvl(iommu, pte, va, 30, PTE_NXTLVL(2)(((2) & 0x7) << 9) | PTE_IR(1LL << 61) | PTE_IW(1LL << 62) | PTE_P(1L << 0));
523 pte = pte_lvl(iommu, pte, va, 21, PTE_NXTLVL(1)(((1) & 0x7) << 9) | PTE_IR(1LL << 61) | PTE_IW(1LL << 62) | PTE_P(1L << 0));
524 if (flags)
525 flags = PTE_P(1L << 0) | PTE_R0x00 | PTE_W(1L << 1) | PTE_IW(1LL << 62) | PTE_IR(1LL << 61) | PTE_NXTLVL(0)(((0) & 0x7) << 9);
526
527 /* Level 1: Page Table - add physical address */
528 idx = (va >> 12) & 0x1FF;
529 pte[idx].val = pa | flags;
530
531 iommu_flush_cache(iommu, pte, sizeof(*pte));
532}
533
534static void
535dmar_dumpseg(bus_dma_tag_t tag, int nseg, bus_dma_segment_t *segs,
536 const char *lbl)
537{
538 struct domain *dom = tag->_cookie;
539 int i;
540
541 return;
542 if (!debugme(dom))
543 return;
544 printf("%s: %s\n", lbl, dom_bdf(dom));
545 for (i = 0; i < nseg; i++) {
546 printf(" %.16llx %.8x\n",
547 (uint64_t)segs[i].ds_addr,
548 (uint32_t)segs[i].ds_len);
549 }
550}
551
552/* Unload mapping */
553void
554domain_unload_map(struct domain *dom, bus_dmamap_t dmam)
555{
556 bus_dma_segment_t *seg;
557 paddr_t base, end, idx;
558 psize_t alen;
559 int i;
560
561 if (iommu_bad(dom->iommu)) {
562 printf("unload map no iommu\n");
563 return;
564 }
565
566 for (i = 0; i < dmam->dm_nsegs; i++) {
567 seg = &dmam->dm_segs[i];
568
569 base = trunc_page(seg->ds_addr)((seg->ds_addr) & ~((1 << 12) - 1));
570 end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE)((((seg->ds_addr + seg->ds_len)+((4096)-1))/(4096))*(4096
));
571 alen = end - base;
572
573 if (debugme(dom)) {
574 printf(" va:%.16llx len:%x\n",
575 (uint64_t)base, (uint32_t)alen);
576 }
577
578 /* Clear PTE */
579 for (idx = 0; idx < alen; idx += VTD_PAGE_SIZE4096)
580 domain_map_page(dom, base + idx, 0, 0);
581
582 if (dom->flag & DOM_NOMAP0x2) {
583 printf("%s: nomap %.16llx\n", dom_bdf(dom), (uint64_t)base);
584 continue;
585 }
586
587 mtx_enter(&dom->exlck);
588 if (extent_free(dom->iovamap, base, alen, EX_NOWAIT0x0000)) {
589 panic("domain_unload_map: extent_free");
590 }
591 mtx_leave(&dom->exlck);
592 }
593}
594
595/* map.segs[x].ds_addr is modified to IOMMU virtual PA */
596void
597domain_load_map(struct domain *dom, bus_dmamap_t map, int flags, int pteflag, const char *fn)
598{
599 bus_dma_segment_t *seg;
600 struct iommu_softc *iommu;
601 paddr_t base, end, idx;
602 psize_t alen;
603 u_long res;
604 int i;
605
606 iommu = dom->iommu;
607 if (!iommu_enabled(iommu)) {
608 /* Lazy enable translation when required */
609 if (iommu_enable_translation(iommu, 1)) {
610 return;
611 }
612 }
613 domain_map_check(dom);
614 for (i = 0; i < map->dm_nsegs; i++) {
615 seg = &map->dm_segs[i];
616
617 base = trunc_page(seg->ds_addr)((seg->ds_addr) & ~((1 << 12) - 1));
618 end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE)((((seg->ds_addr + seg->ds_len)+((4096)-1))/(4096))*(4096
));
619 alen = end - base;
620 res = base;
621
622 if (dom->flag & DOM_NOMAP0x2) {
623 goto nomap;
624 }
625
626 /* Allocate DMA Virtual Address */
627 mtx_enter(&dom->exlck);
628 if (extent_alloc(dom->iovamap, alen, VTD_PAGE_SIZE, 0,extent_alloc_subregion((dom->iovamap), (dom->iovamap)->
ex_start, (dom->iovamap)->ex_end, (alen), (4096), (0), (
map->_dm_boundary), (0x0000), (&res))
629 map->_dm_boundary, EX_NOWAIT, &res)extent_alloc_subregion((dom->iovamap), (dom->iovamap)->
ex_start, (dom->iovamap)->ex_end, (alen), (4096), (0), (
map->_dm_boundary), (0x0000), (&res))) {
630 panic("domain_load_map: extent_alloc");
631 }
632 if (res == -1) {
633 panic("got -1 address");
634 }
635 mtx_leave(&dom->exlck);
636
637 /* Reassign DMA address */
638 seg->ds_addr = res | (seg->ds_addr & VTD_PAGE_MASK0xFFF);
639nomap:
640 if (debugme(dom)) {
641 printf(" LOADMAP: %.16llx %x => %.16llx\n",
642 (uint64_t)seg->ds_addr, (uint32_t)seg->ds_len,
643 (uint64_t)res);
644 }
645 for (idx = 0; idx < alen; idx += VTD_PAGE_SIZE4096) {
646 domain_map_page(dom, res + idx, base + idx,
647 PTE_P(1L << 0) | pteflag);
648 }
649 }
650 if ((iommu->cap & CAP_CM(1LL << 7)) || acpidmar_force_cm) {
651 iommu_flush_tlb(iommu, IOTLB_DOMAIN, dom->did);
652 } else {
653 iommu_flush_write_buffer(iommu);
654 }
655}
656
657const char *
658dom_bdf(struct domain *dom)
659{
660 struct domain_dev *dd;
661 static char mmm[48];
662
663 dd = TAILQ_FIRST(&dom->devices)((&dom->devices)->tqh_first);
664 snprintf(mmm, sizeof(mmm), "%s iommu:%d did:%.4x%s",
665 dmar_bdf(dd->sid), dom->iommu->id, dom->did,
666 dom->did == DID_UNITY0x1 ? " [unity]" : "");
667 return (mmm);
668}
669
670/* Bus DMA Map functions */
671static int
672dmar_dmamap_create(bus_dma_tag_t tag, bus_size_t size, int nsegments,
673 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
674{
675 int rc;
676
677 rc = _bus_dmamap_create(tag, size, nsegments, maxsegsz, boundary,
678 flags, dmamp);
679 if (!rc) {
680 dmar_dumpseg(tag, (*dmamp)->dm_nsegs, (*dmamp)->dm_segs,
681 __FUNCTION__);
682 }
683 return (rc);
684}
685
686static void
687dmar_dmamap_destroy(bus_dma_tag_t tag, bus_dmamap_t dmam)
688{
689 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
690 _bus_dmamap_destroy(tag, dmam);
691}
692
693static int
694dmar_dmamap_load(bus_dma_tag_t tag, bus_dmamap_t dmam, void *buf,
695 bus_size_t buflen, struct proc *p, int flags)
696{
697 struct domain *dom = tag->_cookie;
698 int rc;
699
700 rc = _bus_dmamap_load(tag, dmam, buf, buflen, p, flags);
701 if (!rc) {
702 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
703 __FUNCTION__);
704 domain_load_map(dom, dmam, flags, PTE_R0x00|PTE_W(1L << 1), __FUNCTION__);
705 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
706 __FUNCTION__);
707 }
708 return (rc);
709}
710
711static int
712dmar_dmamap_load_mbuf(bus_dma_tag_t tag, bus_dmamap_t dmam, struct mbuf *chain,
713 int flags)
714{
715 struct domain *dom = tag->_cookie;
716 int rc;
717
718 rc = _bus_dmamap_load_mbuf(tag, dmam, chain, flags);
719 if (!rc) {
720 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
721 __FUNCTION__);
722 domain_load_map(dom, dmam, flags, PTE_R0x00|PTE_W(1L << 1),__FUNCTION__);
723 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
724 __FUNCTION__);
725 }
726 return (rc);
727}
728
729static int
730dmar_dmamap_load_uio(bus_dma_tag_t tag, bus_dmamap_t dmam, struct uio *uio,
731 int flags)
732{
733 struct domain *dom = tag->_cookie;
734 int rc;
735
736 rc = _bus_dmamap_load_uio(tag, dmam, uio, flags);
737 if (!rc) {
738 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
739 __FUNCTION__);
740 domain_load_map(dom, dmam, flags, PTE_R0x00|PTE_W(1L << 1), __FUNCTION__);
741 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
742 __FUNCTION__);
743 }
744 return (rc);
745}
746
747static int
748dmar_dmamap_load_raw(bus_dma_tag_t tag, bus_dmamap_t dmam,
749 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
750{
751 struct domain *dom = tag->_cookie;
752 int rc;
753
754 rc = _bus_dmamap_load_raw(tag, dmam, segs, nsegs, size, flags);
755 if (!rc) {
756 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
757 __FUNCTION__);
758 domain_load_map(dom, dmam, flags, PTE_R0x00|PTE_W(1L << 1), __FUNCTION__);
759 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
760 __FUNCTION__);
761 }
762 return (rc);
763}
764
765static void
766dmar_dmamap_unload(bus_dma_tag_t tag, bus_dmamap_t dmam)
767{
768 struct domain *dom = tag->_cookie;
769
770 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
771 domain_unload_map(dom, dmam);
772 _bus_dmamap_unload(tag, dmam);
773}
774
775static void
776dmar_dmamap_sync(bus_dma_tag_t tag, bus_dmamap_t dmam, bus_addr_t offset,
777 bus_size_t len, int ops)
778{
779#if 0
780 struct domain *dom = tag->_cookie;
781 int flag;
782
783 flag = PTE_P(1L << 0);
784 if (ops == BUS_DMASYNC_PREREAD0x01) {
785 /* make readable */
786 flag |= PTE_R0x00;
787 }
788 else if (ops == BUS_DMASYNC_PREWRITE0x04) {
789 /* make writeable */
790 flag |= PTE_W(1L << 1);
791 }
792 dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
793#endif
794 _bus_dmamap_sync(tag, dmam, offset, len, ops);
795}
796
797static int
798dmar_dmamem_alloc(bus_dma_tag_t tag, bus_size_t size, bus_size_t alignment,
799 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
800 int flags)
801{
802 int rc;
803
804 rc = _bus_dmamem_alloc(tag, size, alignment, boundary, segs, nsegs,
805 rsegs, flags);
806 if (!rc) {
807 dmar_dumpseg(tag, *rsegs, segs, __FUNCTION__);
808 }
809 return (rc);
810}
811
812static void
813dmar_dmamem_free(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs)
814{
815 dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
816 _bus_dmamem_free(tag, segs, nsegs);
817}
818
819static int
820dmar_dmamem_map(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
821 size_t size, caddr_t *kvap, int flags)
822{
823 dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
824 return (_bus_dmamem_map(tag, segs, nsegs, size, kvap, flags));
825}
826
827static void
828dmar_dmamem_unmap(bus_dma_tag_t tag, caddr_t kva, size_t size)
829{
830 struct domain *dom = tag->_cookie;
831
832 if (debugme(dom)) {
833 printf("dmamap_unmap: %s\n", dom_bdf(dom));
834 }
835 _bus_dmamem_unmap(tag, kva, size);
836}
837
838static paddr_t
839dmar_dmamem_mmap(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
840 off_t off, int prot, int flags)
841{
842 dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
843 return (_bus_dmamem_mmap(tag, segs, nsegs, off, prot, flags));
844}
845
846/*===================================
847 * IOMMU code
848 *===================================*/
849
850/* Intel: Set Context Root Address */
851void
852iommu_set_rtaddr(struct iommu_softc *iommu, paddr_t paddr)
853{
854 int i, sts;
855
856 mtx_enter(&iommu->reg_lock);
857 iommu_write_8(iommu, DMAR_RTADDR_REG0x20, paddr);
858 iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd | GCMD_SRTP(1LL << 30));
859 for (i = 0; i < 5; i++) {
860 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
861 if (sts & GSTS_RTPS(1LL << 30))
862 break;
863 }
864 mtx_leave(&iommu->reg_lock);
865
866 if (i == 5) {
867 printf("set_rtaddr fails\n");
868 }
869}
870
871/* Allocate contiguous memory (1Mb) for the Device Table Entries */
872void *
873iommu_alloc_hwdte(struct acpidmar_softc *sc, size_t size, paddr_t *paddr)
874{
875 caddr_t vaddr;
876 bus_dmamap_t map;
877 bus_dma_segment_t seg;
878 bus_dma_tag_t dmat = sc->sc_dmat;
879 int rc, nsegs;
880
881 rc = _bus_dmamap_create(dmat, size, 1, size, 0,
882 BUS_DMA_NOWAIT0x0001, &map);
883 if (rc != 0) {
884 printf("hwdte_create fails\n");
885 return NULL((void *)0);
886 }
887 rc = _bus_dmamem_alloc(dmat, size, 4, 0, &seg, 1,
888 &nsegs, BUS_DMA_NOWAIT0x0001 | BUS_DMA_ZERO0x1000);
889 if (rc != 0) {
890 printf("hwdte alloc fails\n");
891 return NULL((void *)0);
892 }
893 rc = _bus_dmamem_map(dmat, &seg, 1, size, &vaddr,
894 BUS_DMA_NOWAIT0x0001 | BUS_DMA_COHERENT0x0004);
895 if (rc != 0) {
896 printf("hwdte map fails\n");
897 return NULL((void *)0);
898 }
899 rc = _bus_dmamap_load_raw(dmat, map, &seg, 1, size, BUS_DMA_NOWAIT0x0001);
900 if (rc != 0) {
901 printf("hwdte load raw fails\n");
902 return NULL((void *)0);
903 }
904 *paddr = map->dm_segs[0].ds_addr;
905 return vaddr;
906}
907
908/* COMMON: Allocate a new memory page */
909void *
910iommu_alloc_page(struct iommu_softc *iommu, paddr_t *paddr)
911{
912 void *va;
913
914 *paddr = 0;
915 va = km_alloc(VTD_PAGE_SIZE4096, &kv_page, &kp_zero, &kd_nowait);
916 if (va == NULL((void *)0)) {
917 panic("can't allocate page");
918 }
919 pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)va, paddr);
920 return (va);
921}
922
923
924/* Intel: Issue command via queued invalidation */
925void
926iommu_issue_qi(struct iommu_softc *iommu, struct qi_entry *qi)
927{
928#if 0
929 struct qi_entry *pi, *pw;
930
931 idx = iommu->qi_head;
932 pi = &iommu->qi[idx];
933 pw = &iommu->qi[(idx+1) % MAXQ];
934 iommu->qi_head = (idx+2) % MAXQ;
935
936 memcpy(pw, &qi, sizeof(qi))__builtin_memcpy((pw), (&qi), (sizeof(qi)));
937 issue command;
938 while (pw->xxx)
939 ;
940#endif
941}
942
943/* Intel: Flush TLB entries, Queued Invalidation mode */
944void
945iommu_flush_tlb_qi(struct iommu_softc *iommu, int mode, int did)
946{
947 struct qi_entry qi;
948
949 /* Use queued invalidation */
950 qi.hi = 0;
951 switch (mode) {
1
'Default' branch taken. Execution continues on line 964
→
952 case IOTLB_GLOBAL:
953 qi.lo = QI_IOTLB0x2 | QI_IOTLB_IG_GLOBAL(1 << 4);
954 break;
955 case IOTLB_DOMAIN:
956 qi.lo = QI_IOTLB0x2 | QI_IOTLB_IG_DOMAIN(2 << 4) |
957 QI_IOTLB_DID(did)(((uint64_t)(did) << 16));
958 break;
959 case IOTLB_PAGE:
960 qi.lo = QI_IOTLB0x2 | QI_IOTLB_IG_PAGE(3 << 4) | QI_IOTLB_DID(did)(((uint64_t)(did) << 16));
961 qi.hi = 0;
962 break;
963 }
964 if (iommu->cap & CAP_DRD(1LL << 55))
2
←
Assuming the condition is false
→
3
←
Taking false branch
→
965 qi.lo |= QI_IOTLB_DR(1LL << 6);
966 if (iommu->cap & CAP_DWD(1LL << 54))
4
←
Assuming the condition is true
→
5
←
Taking true branch
→
967 qi.lo |= QI_IOTLB_DW(1LL << 5);
6
←
The left expression of the compound assignment is an uninitialized value. The computed value will also be garbage
968 iommu_issue_qi(iommu, &qi);
969}
970
971/* Intel: Flush Context entries, Queued Invalidation mode */
972void
973iommu_flush_ctx_qi(struct iommu_softc *iommu, int mode, int did,
974 int sid, int fm)
975{
976 struct qi_entry qi;
977
978 /* Use queued invalidation */
979 qi.hi = 0;
980 switch (mode) {
981 case CTX_GLOBAL:
982 qi.lo = QI_CTX0x1 | QI_CTX_IG_GLOBAL(CTX_GLOBAL << 4);
983 break;
984 case CTX_DOMAIN:
985 qi.lo = QI_CTX0x1 | QI_CTX_IG_DOMAIN(CTX_DOMAIN << 4) | QI_CTX_DID(did)(((uint64_t)(did) << 16));
986 break;
987 case CTX_DEVICE:
988 qi.lo = QI_CTX0x1 | QI_CTX_IG_DEVICE(CTX_DEVICE << 4) | QI_CTX_DID(did)(((uint64_t)(did) << 16)) |
989 QI_CTX_SID(sid)(((uint64_t)(sid) << 32)) | QI_CTX_FM(fm)(((uint64_t)(fm) << 48));
990 break;
991 }
992 iommu_issue_qi(iommu, &qi);
993}
994
995/* Intel: Flush write buffers */
996void
997iommu_flush_write_buffer(struct iommu_softc *iommu)
998{
999 int i, sts;
1000
1001 if (iommu->dte)
1002 return;
1003 if (!(iommu->cap & CAP_RWBF(1LL << 4)))
1004 return;
1005 DPRINTF(1,"writebuf\n");
1006 iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd | GCMD_WBF(1LL << 27));
1007 for (i = 0; i < 5; i++) {
1008 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1009 if (sts & GSTS_WBFS(1LL << 27))
1010 break;
1011 delay(10000)(*delay_func)(10000);
1012 }
1013 if (i == 5) {
1014 printf("write buffer flush fails\n");
1015 }
1016}
1017
1018void
1019iommu_flush_cache(struct iommu_softc *iommu, void *addr, size_t size)
1020{
1021 if (iommu->dte) {
1022 pmap_flush_cache((vaddr_t)addr, size);
1023 return;
1024 }
1025 if (!(iommu->ecap & ECAP_C(1LL << 0)))
1026 pmap_flush_cache((vaddr_t)addr, size);
1027}
1028
1029/*
1030 * Intel: Flush IOMMU TLB Entries
1031 * Flushing can occur globally, per domain or per page
1032 */
1033void
1034iommu_flush_tlb(struct iommu_softc *iommu, int mode, int did)
1035{
1036 int n;
1037 uint64_t val;
1038
1039 /* Call AMD */
1040 if (iommu->dte) {
1041 ivhd_invalidate_domain(iommu, did);
1042 return;
1043 }
1044 val = IOTLB_IVT(1LL << 63);
1045 switch (mode) {
1046 case IOTLB_GLOBAL:
1047 val |= IIG_GLOBAL((uint64_t)(IOTLB_GLOBAL) << 60);
1048 break;
1049 case IOTLB_DOMAIN:
1050 val |= IIG_DOMAIN((uint64_t)(IOTLB_DOMAIN) << 60) | IOTLB_DID(did)((uint64_t)(did) << 32);
1051 break;
1052 case IOTLB_PAGE:
1053 val |= IIG_PAGE((uint64_t)(IOTLB_PAGE) << 60) | IOTLB_DID(did)((uint64_t)(did) << 32);
1054 break;
1055 }
1056
1057 /* Check for Read/Write Drain */
1058 if (iommu->cap & CAP_DRD(1LL << 55))
1059 val |= IOTLB_DR(1LL << 49);
1060 if (iommu->cap & CAP_DWD(1LL << 54))
1061 val |= IOTLB_DW(1LL << 48);
1062
1063 mtx_enter(&iommu->reg_lock);
1064
1065 iommu_write_8(iommu, DMAR_IOTLB_REG(iommu)(((uint32_t)((((iommu)->ecap)>> 0x8) & 0x3FF) * 16
) + 8), val);
1066 n = 0;
1067 do {
1068 val = iommu_read_8(iommu, DMAR_IOTLB_REG(iommu)(((uint32_t)((((iommu)->ecap)>> 0x8) & 0x3FF) * 16
) + 8));
1069 } while (n++ < 5 && val & IOTLB_IVT(1LL << 63));
1070
1071 mtx_leave(&iommu->reg_lock);
1072}
1073
1074/* Intel: Flush IOMMU settings
1075 * Flushes can occur globally, per domain, or per device
1076 */
1077void
1078iommu_flush_ctx(struct iommu_softc *iommu, int mode, int did, int sid, int fm)
1079{
1080 uint64_t val;
1081 int n;
1082
1083 if (iommu->dte)
1084 return;
1085 val = CCMD_ICC(1LL << 63);
1086 switch (mode) {
1087 case CTX_GLOBAL:
1088 val |= CIG_GLOBAL((uint64_t)(CTX_GLOBAL) << 61);
1089 break;
1090 case CTX_DOMAIN:
1091 val |= CIG_DOMAIN((uint64_t)(CTX_DOMAIN) << 61) | CCMD_DID(did)(((did) << 0));
1092 break;
1093 case CTX_DEVICE:
1094 val |= CIG_DEVICE((uint64_t)(CTX_DEVICE) << 61) | CCMD_DID(did)(((did) << 0)) |
1095 CCMD_SID(sid)(((sid) << 8)) | CCMD_FM(fm)(((uint64_t)(fm) << 32));
1096 break;
1097 }
1098
1099 mtx_enter(&iommu->reg_lock);
1100
1101 n = 0;
1102 iommu_write_8(iommu, DMAR_CCMD_REG0x28, val);
1103 do {
1104 val = iommu_read_8(iommu, DMAR_CCMD_REG0x28);
1105 } while (n++ < 5 && val & CCMD_ICC(1LL << 63));
1106
1107 mtx_leave(&iommu->reg_lock);
1108}
1109
1110/* Intel: Enable Queued Invalidation */
1111void
1112iommu_enable_qi(struct iommu_softc *iommu, int enable)
1113{
1114 int n = 0;
1115 int sts;
1116
1117 if (!(iommu->ecap & ECAP_QI(1LL << 1)))
1118 return;
1119
1120 if (enable) {
1121 iommu->gcmd |= GCMD_QIE(1LL << 26);
1122
1123 mtx_enter(&iommu->reg_lock);
1124
1125 iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1126 do {
1127 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1128 } while (n++ < 5 && !(sts & GSTS_QIES(1LL << 26)));
1129
1130 mtx_leave(&iommu->reg_lock);
1131
1132 DPRINTF(1,"set.qie: %d\n", n);
1133 } else {
1134 iommu->gcmd &= ~GCMD_QIE(1LL << 26);
1135
1136 mtx_enter(&iommu->reg_lock);
1137
1138 iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1139 do {
1140 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1141 } while (n++ < 5 && sts & GSTS_QIES(1LL << 26));
1142
1143 mtx_leave(&iommu->reg_lock);
1144
1145 DPRINTF(1,"clr.qie: %d\n", n);
1146 }
1147}
1148
1149/* Intel: Enable IOMMU translation */
1150int
1151iommu_enable_translation(struct iommu_softc *iommu, int enable)
1152{
1153 uint32_t sts;
1154 uint64_t reg;
1155 int n = 0;
1156
1157 if (iommu->dte)
1158 return (0);
1159 reg = 0;
1160 if (enable) {
1161 DPRINTF(0,"enable iommu %d\n", iommu->id);
1162 iommu_showcfg(iommu, -1);
1163
1164 iommu->gcmd |= GCMD_TE(1LL << 31);
1165
1166 /* Enable translation */
1167 printf(" pre tes: ");
1168
1169 mtx_enter(&iommu->reg_lock);
1170 iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1171 printf("xxx");
1172 do {
1173 printf("yyy");
1174 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1175 delay(n * 10000)(*delay_func)(n * 10000);
1176 } while (n++ < 5 && !(sts & GSTS_TES(1LL << 31)));
1177 mtx_leave(&iommu->reg_lock);
1178
1179 printf(" set.tes: %d\n", n);
1180
1181 if (n >= 5) {
1182 printf("error.. unable to initialize iommu %d\n",
1183 iommu->id);
1184 iommu->flags |= IOMMU_FLAGS_BAD0x2;
1185
1186 /* Disable IOMMU */
1187 iommu->gcmd &= ~GCMD_TE(1LL << 31);
1188 mtx_enter(&iommu->reg_lock);
1189 iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1190 mtx_leave(&iommu->reg_lock);
1191
1192 return (1);
1193 }
1194
1195 iommu_flush_ctx(iommu, CTX_GLOBAL, 0, 0, 0);
1196 iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
1197 } else {
1198 iommu->gcmd &= ~GCMD_TE(1LL << 31);
1199
1200 mtx_enter(&iommu->reg_lock);
1201
1202 iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1203 do {
1204 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1205 } while (n++ < 5 && sts & GSTS_TES(1LL << 31));
1206 mtx_leave(&iommu->reg_lock);
1207
1208 printf(" clr.tes: %d\n", n);
1209 }
1210
1211 return (0);
1212}
1213
1214/* Intel: Initialize IOMMU */
1215int
1216iommu_init(struct acpidmar_softc *sc, struct iommu_softc *iommu,
1217 struct acpidmar_drhd *dh)
1218{
1219 static int niommu;
1220 int len = VTD_PAGE_SIZE4096;
1221 int i, gaw;
1222 uint32_t sts;
1223 paddr_t paddr;
1224
1225 if (_bus_space_map(sc->sc_memt, dh->address, len, 0, &iommu->ioh) != 0) {
1226 return (-1);
1227 }
1228
1229 TAILQ_INIT(&iommu->domains)do { (&iommu->domains)->tqh_first = ((void *)0); (&
iommu->domains)->tqh_last = &(&iommu->domains
)->tqh_first; } while (0);
1230 iommu->id = ++niommu;
1231 iommu->flags = dh->flags;
1232 iommu->segment = dh->segment;
1233 iommu->iot = sc->sc_memt;
1234
1235 iommu->cap = iommu_read_8(iommu, DMAR_CAP_REG0x08);
1236 iommu->ecap = iommu_read_8(iommu, DMAR_ECAP_REG0x10);
1237 iommu->ndoms = cap_nd(iommu->cap)(16 << (((iommu->cap) & 0x7) << 1));
1238
1239 /* Print Capabilities & Extended Capabilities */
1240 DPRINTF(0, " caps: %s%s%s%s%s%s%s%s%s%s%s\n",
1241 iommu->cap & CAP_AFL ? "afl " : "", /* adv fault */
1242 iommu->cap & CAP_RWBF ? "rwbf " : "", /* write-buffer flush */
1243 iommu->cap & CAP_PLMR ? "plmr " : "", /* protected lo region */
1244 iommu->cap & CAP_PHMR ? "phmr " : "", /* protected hi region */
1245 iommu->cap & CAP_CM ? "cm " : "", /* caching mode */
1246 iommu->cap & CAP_ZLR ? "zlr " : "", /* zero-length read */
1247 iommu->cap & CAP_PSI ? "psi " : "", /* page invalidate */
1248 iommu->cap & CAP_DWD ? "dwd " : "", /* write drain */
1249 iommu->cap & CAP_DRD ? "drd " : "", /* read drain */
1250 iommu->cap & CAP_FL1GP ? "Gb " : "", /* 1Gb pages */
1251 iommu->cap & CAP_PI ? "pi " : ""); /* posted interrupts */
1252 DPRINTF(0, " ecap: %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
1253 iommu->ecap & ECAP_C ? "c " : "", /* coherent */
1254 iommu->ecap & ECAP_QI ? "qi " : "", /* queued invalidate */
1255 iommu->ecap & ECAP_DT ? "dt " : "", /* device iotlb */
1256 iommu->ecap & ECAP_IR ? "ir " : "", /* intr remap */
1257 iommu->ecap & ECAP_EIM ? "eim " : "", /* x2apic */
1258 iommu->ecap & ECAP_PT ? "pt " : "", /* passthrough */
1259 iommu->ecap & ECAP_SC ? "sc " : "", /* snoop control */
1260 iommu->ecap & ECAP_ECS ? "ecs " : "", /* extended context */
1261 iommu->ecap & ECAP_MTS ? "mts " : "", /* memory type */
1262 iommu->ecap & ECAP_NEST ? "nest " : "", /* nested translations */
1263 iommu->ecap & ECAP_DIS ? "dis " : "", /* deferred invalidation */
1264 iommu->ecap & ECAP_PASID ? "pas " : "", /* pasid */
1265 iommu->ecap & ECAP_PRS ? "prs " : "", /* page request */
1266 iommu->ecap & ECAP_ERS ? "ers " : "", /* execute request */
1267 iommu->ecap & ECAP_SRS ? "srs " : "", /* supervisor request */
1268 iommu->ecap & ECAP_NWFS ? "nwfs " : "", /* no write flag */
1269 iommu->ecap & ECAP_EAFS ? "eafs " : ""); /* extended accessed flag */
1270
1271 mtx_init(&iommu->reg_lock, IPL_HIGH)do { (void)(((void *)0)); (void)(0); __mtx_init((&iommu->
reg_lock), ((((0xd)) > 0x0 && ((0xd)) < 0x9) ? 0x9
: ((0xd)))); } while (0);
1272
1273 /* Clear Interrupt Masking */
1274 iommu_write_4(iommu, DMAR_FSTS_REG0x34, FSTS_PFO(1LL << 0) | FSTS_PPF(1LL << 1));
1275
1276 iommu->intr = acpidmar_intr_establish(iommu, IPL_HIGH0xd,
1277 acpidmar_intr, iommu, "dmarintr");
1278
1279 /* Enable interrupts */
1280 sts = iommu_read_4(iommu, DMAR_FECTL_REG0x38);
1281 iommu_write_4(iommu, DMAR_FECTL_REG0x38, sts & ~FECTL_IM(1LL << 31));
1282
1283 /* Allocate root pointer */
1284 iommu->root = iommu_alloc_page(iommu, &paddr);
1285 DPRINTF(0, "Allocated root pointer: pa:%.16llx va:%p\n",
1286 (uint64_t)paddr, iommu->root);
1287 iommu->rtaddr = paddr;
1288 iommu_flush_write_buffer(iommu);
1289 iommu_set_rtaddr(iommu, paddr);
1290
1291#if 0
1292 if (iommu->ecap & ECAP_QI(1LL << 1)) {
1293 /* Queued Invalidation support */
1294 iommu->qi = iommu_alloc_page(iommu, &iommu->qip);
1295 iommu_write_8(iommu, DMAR_IQT_REG0x88, 0);
1296 iommu_write_8(iommu, DMAR_IQA_REG0x90, iommu->qip | IQA_QS_2560);
1297 }
1298 if (iommu->ecap & ECAP_IR(1LL << 3)) {
1299 /* Interrupt remapping support */
1300 iommu_write_8(iommu, DMAR_IRTA_REG0xb8, 0);
1301 }
1302#endif
1303
1304 /* Calculate guest address width and supported guest widths */
1305 gaw = -1;
1306 iommu->mgaw = cap_mgaw(iommu->cap)((uint32_t)(((iommu->cap)>> 16LL) & 0x3F) + 1);
1307 DPRINTF(0, "gaw: %d { ", iommu->mgaw);
1308 for (i = 0; i < 5; i++) {
1309 if (cap_sagaw(iommu->cap)(uint32_t)(((iommu->cap)>> 8LL) & 0x1F) & (1L << i)) {
1310 gaw = VTD_LEVELTOAW(i)(((i) * 9) + 30);
1311 DPRINTF(0, "%d ", gaw);
1312 iommu->agaw = gaw;
1313 }
1314 }
1315 DPRINTF(0, "}\n");
1316
1317 /* Cache current status register bits */
1318 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1319 if (sts & GSTS_TES(1LL << 31))
1320 iommu->gcmd |= GCMD_TE(1LL << 31);
1321 if (sts & GSTS_QIES(1LL << 26))
1322 iommu->gcmd |= GCMD_QIE(1LL << 26);
1323 if (sts & GSTS_IRES(1LL << 25))
1324 iommu->gcmd |= GCMD_IRE(1LL << 25);
1325 DPRINTF(0, "gcmd: %x preset\n", iommu->gcmd);
1326 acpidmar_intr(iommu);
1327 return (0);
1328}
1329
1330/* Read/Write IOMMU register */
1331uint32_t
1332iommu_read_4(struct iommu_softc *iommu, int reg)
1333{
1334 uint32_t v;
1335
1336 v = bus_space_read_4(iommu->iot, iommu->ioh, reg)((iommu->iot)->read_4((iommu->ioh), (reg)));
1337 return (v);
1338}
1339
1340
1341void
1342iommu_write_4(struct iommu_softc *iommu, int reg, uint32_t v)
1343{
1344 bus_space_write_4(iommu->iot, iommu->ioh, reg, (uint32_t)v)((iommu->iot)->write_4((iommu->ioh), (reg), ((uint32_t
)v)));
1345}
1346
1347uint64_t
1348iommu_read_8(struct iommu_softc *iommu, int reg)
1349{
1350 uint64_t v;
1351
1352 v = bus_space_read_8(iommu->iot, iommu->ioh, reg)((iommu->iot)->read_8((iommu->ioh), (reg)));
1353 return (v);
1354}
1355
1356void
1357iommu_write_8(struct iommu_softc *iommu, int reg, uint64_t v)
1358{
1359 bus_space_write_8(iommu->iot, iommu->ioh, reg, v)((iommu->iot)->write_8((iommu->ioh), (reg), (v)));
1360}
1361
1362/* Check if a device is within a device scope */
1363int
1364acpidmar_match_devscope(struct devlist_head *devlist, pci_chipset_tag_t pc,
1365 int sid)
1366{
1367 struct dmar_devlist *ds;
1368 int sub, sec, i;
1369 int bus, dev, fun, sbus;
1370 pcireg_t reg;
1371 pcitag_t tag;
1372
1373 sbus = sid_bus(sid);
1374 TAILQ_FOREACH(ds, devlist, link)for((ds) = ((devlist)->tqh_first); (ds) != ((void *)0); (ds
) = ((ds)->link.tqe_next)) {
1375 bus = ds->bus;
1376 dev = ds->dp[0].device;
1377 fun = ds->dp[0].function;
1378 /* Walk PCI bridges in path */
1379 for (i = 1; i < ds->ndp; i++) {
1380 tag = pci_make_tag(pc, bus, dev, fun);
1381 reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO0x18);
1382 bus = PPB_BUSINFO_SECONDARY(reg)((reg >> 8) & 0xff);
1383 dev = ds->dp[i].device;
1384 fun = ds->dp[i].function;
1385 }
1386
1387 /* Check for device exact match */
1388 if (sid == mksid(bus, dev, fun)) {
1389 return DMAR_ENDPOINT0x1;
1390 }
1391
1392 /* Check for device subtree match */
1393 if (ds->type == DMAR_BRIDGE0x2) {
1394 tag = pci_make_tag(pc, bus, dev, fun);
1395 reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO0x18);
1396 sec = PPB_BUSINFO_SECONDARY(reg)((reg >> 8) & 0xff);
1397 sub = PPB_BUSINFO_SUBORDINATE(reg)((reg >> 16) & 0xff);
1398 if (sec <= sbus && sbus <= sub) {
1399 return DMAR_BRIDGE0x2;
1400 }
1401 }
1402 }
1403
1404 return (0);
1405}
1406
1407struct domain *
1408domain_create(struct iommu_softc *iommu, int did)
1409{
1410 struct domain *dom;
1411 int gaw;
1412
1413 DPRINTF(0, "iommu%d: create domain: %.4x\n", iommu->id, did);
1414 dom = malloc(sizeof(*dom), M_DEVBUF2, M_ZERO0x0008 | M_WAITOK0x0001);
1415 dom->did = did;
1416 dom->iommu = iommu;
1417 dom->pte = iommu_alloc_page(iommu, &dom->ptep);
1418 TAILQ_INIT(&dom->devices)do { (&dom->devices)->tqh_first = ((void *)0); (&
dom->devices)->tqh_last = &(&dom->devices)->
tqh_first; } while (0);
1419
1420 /* Setup DMA */
1421 dom->dmat._cookie = dom;
1422 dom->dmat._dmamap_create = dmar_dmamap_create; /* nop */
1423 dom->dmat._dmamap_destroy = dmar_dmamap_destroy; /* nop */
1424 dom->dmat._dmamap_load = dmar_dmamap_load; /* lm */
1425 dom->dmat._dmamap_load_mbuf = dmar_dmamap_load_mbuf; /* lm */
1426 dom->dmat._dmamap_load_uio = dmar_dmamap_load_uio; /* lm */
1427 dom->dmat._dmamap_load_raw = dmar_dmamap_load_raw; /* lm */
1428 dom->dmat._dmamap_unload = dmar_dmamap_unload; /* um */
1429 dom->dmat._dmamap_sync = dmar_dmamap_sync; /* lm */
1430 dom->dmat._dmamem_alloc = dmar_dmamem_alloc; /* nop */
1431 dom->dmat._dmamem_free = dmar_dmamem_free; /* nop */
1432 dom->dmat._dmamem_map = dmar_dmamem_map; /* nop */
1433 dom->dmat._dmamem_unmap = dmar_dmamem_unmap; /* nop */
1434 dom->dmat._dmamem_mmap = dmar_dmamem_mmap;
1435
1436 snprintf(dom->exname, sizeof(dom->exname), "did:%x.%.4x",
1437 iommu->id, dom->did);
1438
1439 /* Setup IOMMU address map */
1440 gaw = min(iommu->agaw, iommu->mgaw);
1441 dom->iovamap = extent_create(dom->exname, 0, (1LL << gaw)-1,
1442 M_DEVBUF2, NULL((void *)0), 0, EX_WAITOK0x0001 | EX_NOCOALESCE0x0008);
1443
1444 /* Reserve the first 16M */
1445 extent_alloc_region(dom->iovamap, 0, 16*1024*1024, EX_WAITOK0x0001);
1446
1447 /* Zero out MSI Interrupt region */
1448 extent_alloc_region(dom->iovamap, MSI_BASE_ADDRESS0xFEE00000L, MSI_BASE_SIZE0x00100000L,
1449 EX_WAITOK0x0001);
1450 mtx_init(&dom->exlck, IPL_HIGH)do { (void)(((void *)0)); (void)(0); __mtx_init((&dom->
exlck), ((((0xd)) > 0x0 && ((0xd)) < 0x9) ? 0x9
: ((0xd)))); } while (0);
1451
1452 TAILQ_INSERT_TAIL(&iommu->domains, dom, link)do { (dom)->link.tqe_next = ((void *)0); (dom)->link.tqe_prev
= (&iommu->domains)->tqh_last; *(&iommu->domains
)->tqh_last = (dom); (&iommu->domains)->tqh_last
= &(dom)->link.tqe_next; } while (0);
1453
1454 return dom;
1455}
1456
1457void
1458domain_add_device(struct domain *dom, int sid)
1459{
1460 struct domain_dev *ddev;
1461
1462 DPRINTF(0, "add %s to iommu%d.%.4x\n", dmar_bdf(sid), dom->iommu->id, dom->did);
1463 ddev = malloc(sizeof(*ddev), M_DEVBUF2, M_ZERO0x0008 | M_WAITOK0x0001);
1464 ddev->sid = sid;
1465 TAILQ_INSERT_TAIL(&dom->devices, ddev, link)do { (ddev)->link.tqe_next = ((void *)0); (ddev)->link.
tqe_prev = (&dom->devices)->tqh_last; *(&dom->
devices)->tqh_last = (ddev); (&dom->devices)->tqh_last
= &(ddev)->link.tqe_next; } while (0);
1466
1467 /* Should set context entry here?? */
1468}
1469
1470void
1471domain_remove_device(struct domain *dom, int sid)
1472{
1473 struct domain_dev *ddev, *tmp;
1474
1475 TAILQ_FOREACH_SAFE(ddev, &dom->devices, link, tmp)for ((ddev) = ((&dom->devices)->tqh_first); (ddev) !=
((void *)0) && ((tmp) = ((ddev)->link.tqe_next), 1
); (ddev) = (tmp)) {
1476 if (ddev->sid == sid) {
1477 TAILQ_REMOVE(&dom->devices, ddev, link)do { if (((ddev)->link.tqe_next) != ((void *)0)) (ddev)->
link.tqe_next->link.tqe_prev = (ddev)->link.tqe_prev; else
(&dom->devices)->tqh_last = (ddev)->link.tqe_prev
; *(ddev)->link.tqe_prev = (ddev)->link.tqe_next; ((ddev
)->link.tqe_prev) = ((void *)-1); ((ddev)->link.tqe_next
) = ((void *)-1); } while (0);
1478 free(ddev, sizeof(*ddev), M_DEVBUF2);
1479 }
1480 }
1481}
1482
1483/* Lookup domain by segment & source id (bus.device.function) */
1484struct domain *
1485domain_lookup(struct acpidmar_softc *sc, int segment, int sid)
1486{
1487 struct iommu_softc *iommu;
1488 struct domain_dev *ddev;
1489 struct domain *dom;
1490 int rc;
1491
1492 if (sc == NULL((void *)0)) {
1493 return NULL((void *)0);
1494 }
1495
1496 /* Lookup IOMMU for this device */
1497 TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu)
!= ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
1498 if (iommu->segment != segment)
1499 continue;
1500 /* Check for devscope match or catchall iommu */
1501 rc = acpidmar_match_devscope(&iommu->devices, sc->sc_pc, sid);
1502 if (rc != 0 || iommu->flags) {
1503 break;
1504 }
1505 }
1506 if (!iommu) {
1507 printf("%s: no iommu found\n", dmar_bdf(sid));
1508 return NULL((void *)0);
1509 }
1510
1511 /* Search domain devices */
1512 TAILQ_FOREACH(dom, &iommu->domains, link)for((dom) = ((&iommu->domains)->tqh_first); (dom) !=
((void *)0); (dom) = ((dom)->link.tqe_next)) {
1513 TAILQ_FOREACH(ddev, &dom->devices, link)for((ddev) = ((&dom->devices)->tqh_first); (ddev) !=
((void *)0); (ddev) = ((ddev)->link.tqe_next)) {
1514 /* XXX: match all functions? */
1515 if (ddev->sid == sid) {
1516 return dom;
1517 }
1518 }
1519 }
1520 if (iommu->ndoms <= 2) {
1521 /* Running out of domains.. create catchall domain */
1522 if (!iommu->unity) {
1523 iommu->unity = domain_create(iommu, 1);
1524 }
1525 dom = iommu->unity;
1526 } else {
1527 dom = domain_create(iommu, --iommu->ndoms);
1528 }
1529 if (!dom) {
1530 printf("no domain here\n");
1531 return NULL((void *)0);
1532 }
1533
1534 /* Add device to domain */
1535 domain_add_device(dom, sid);
1536
1537 return dom;
1538}
1539
1540/* Map Guest Pages into IOMMU */
1541void
1542_iommu_map(void *dom, vaddr_t va, bus_addr_t gpa, bus_size_t len)
1543{
1544 bus_size_t i;
1545 paddr_t hpa;
1546
1547 if (dom == NULL((void *)0)) {
1548 return;
1549 }
1550 DPRINTF(1, "Mapping dma: %lx = %lx/%lx\n", va, gpa, len);
1551 for (i = 0; i < len; i += PAGE_SIZE(1 << 12)) {
1552 hpa = 0;
1553 pmap_extract(curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_vmspace->vm_map.pmap, va, &hpa);
1554 domain_map_page(dom, gpa, hpa, PTE_P(1L << 0) | PTE_R0x00 | PTE_W(1L << 1));
1555 gpa += PAGE_SIZE(1 << 12);
1556 va += PAGE_SIZE(1 << 12);
1557 }
1558}
1559
1560/* Find IOMMU for a given PCI device */
1561void
1562*_iommu_domain(int segment, int bus, int dev, int func, int *id)
1563{
1564 struct domain *dom;
1565
1566 dom = domain_lookup(acpidmar_sc, segment, mksid(bus, dev, func));
1567 if (dom) {
1568 *id = dom->did;
1569 }
1570 return dom;
1571}
1572
1573void
1574domain_map_device(struct domain *dom, int sid);
1575
1576void
1577domain_map_device(struct domain *dom, int sid)
1578{
1579 struct iommu_softc *iommu;
1580 struct context_entry *ctx;
1581 paddr_t paddr;
1582 int bus, devfn;
1583 int tt, lvl;
1584
1585 iommu = dom->iommu;
1586
1587 bus = sid_bus(sid);
1588 devfn = sid_devfn(sid);
1589 /* AMD attach device */
1590 if (iommu->dte) {
1591 struct ivhd_dte *dte = &iommu->dte[sid];
1592 if (!dte->dw0) {
1593 /* Setup Device Table Entry: bus.devfn */
1594 DPRINTF(1, "@@@ PCI Attach: %.4x[%s] %.4x\n", sid, dmar_bdf(sid), dom->did);
1595 dte_set_host_page_table_root_ptr(dte, dom->ptep);
1596 dte_set_domain(dte, dom->did);
1597 dte_set_mode(dte, 3); /* Set 3 level PTE */
1598 dte_set_tv(dte);
1599 dte_set_valid(dte);
1600 ivhd_flush_devtab(iommu, dom->did);
1601#ifdef IOMMU_DEBUG
1602 //ivhd_showreg(iommu);
1603 ivhd_showdte(iommu);
1604#endif
1605 }
1606 return;
1607 }
1608
1609 /* Create Bus mapping */
1610 if (!root_entry_is_valid(&iommu->root[bus])) {
1611 iommu->ctx[bus] = iommu_alloc_page(iommu, &paddr);
1612 iommu->root[bus].lo = paddr | ROOT_P(1L << 0);
1613 iommu_flush_cache(iommu, &iommu->root[bus],
1614 sizeof(struct root_entry));
1615 DPRINTF(0, "iommu%d: Allocate context for bus: %.2x pa:%.16llx va:%p\n",
1616 iommu->id, bus, (uint64_t)paddr,
1617 iommu->ctx[bus]);
1618 }
1619
1620 /* Create DevFn mapping */
1621 ctx = iommu->ctx[bus] + devfn;
1622 if (!context_entry_is_valid(ctx)) {
1623 tt = CTX_T_MULTI;
1624 lvl = VTD_AWTOLEVEL(iommu->agaw)(((iommu->agaw) - 30) / 9);
1625
1626 /* Initialize context */
1627 context_set_slpte(ctx, dom->ptep);
1628 context_set_translation_type(ctx, tt);
1629 context_set_domain_id(ctx, dom->did);
1630 context_set_address_width(ctx, lvl);
1631 context_set_present(ctx);
1632
1633 /* Flush it */
1634 iommu_flush_cache(iommu, ctx, sizeof(struct context_entry));
1635 if ((iommu->cap & CAP_CM(1LL << 7)) || acpidmar_force_cm) {
1636 iommu_flush_ctx(iommu, CTX_DEVICE, dom->did, sid, 0);
1637 iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
1638 } else {
1639 iommu_flush_write_buffer(iommu);
1640 }
1641 DPRINTF(0, "iommu%d: %s set context ptep:%.16llx lvl:%d did:%.4x tt:%d\n",
1642 iommu->id, dmar_bdf(sid), (uint64_t)dom->ptep, lvl,
1643 dom->did, tt);
1644 }
1645}
1646
1647struct domain *
1648acpidmar_pci_attach(struct acpidmar_softc *sc, int segment, int sid, int mapctx)
1649{
1650 static struct domain *dom;
1651
1652 dom = domain_lookup(sc, segment, sid);
1653 if (!dom) {
1654 printf("no domain: %s\n", dmar_bdf(sid));
1655 return NULL((void *)0);
1656 }
1657
1658 if (mapctx) {
1659 domain_map_device(dom, sid);
1660 }
1661
1662 return dom;
1663}
1664
1665void
1666acpidmar_pci_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
1667{
1668 int bus, dev, fun, sid;
1669 struct domain *dom;
1670 pcireg_t reg;
1671
1672 if (!acpidmar_sc) {
1673 /* No DMAR, ignore */
1674 return;
1675 }
1676
1677 /* Add device to our list if valid */
1678 pci_decompose_tag(pc, pa->pa_tag, &bus, &dev, &fun);
1679 sid = mksid(bus, dev, fun);
1680 if (sid_flag[sid] & SID_INVALID0x80000000L)
1681 return;
1682
1683 reg = pci_conf_read(pc, pa->pa_tag, PCI_CLASS_REG0x08);
1684
1685 /* Add device to domain */
1686 dom = acpidmar_pci_attach(acpidmar_sc, pa->pa_domain, sid, 0);
1687 if (dom == NULL((void *)0))
1688 return;
1689
1690 if (PCI_CLASS(reg)(((reg) >> 24) & 0xff) == PCI_CLASS_DISPLAY0x03 &&
1691 PCI_SUBCLASS(reg)(((reg) >> 16) & 0xff) == PCI_SUBCLASS_DISPLAY_VGA0x00) {
1692 dom->flag = DOM_NOMAP0x2;
1693 }
1694 if (PCI_CLASS(reg)(((reg) >> 24) & 0xff) == PCI_CLASS_BRIDGE0x06 &&
1695 PCI_SUBCLASS(reg)(((reg) >> 16) & 0xff) == PCI_SUBCLASS_BRIDGE_ISA0x01) {
1696 /* For ISA Bridges, map 0-16Mb as 1:1 */
1697 printf("dmar: %.4x:%.2x:%.2x.%x mapping ISA\n",
1698 pa->pa_domain, bus, dev, fun);
1699 domain_map_pthru(dom, 0x00, 16*1024*1024);
1700 }
1701
1702 /* Change DMA tag */
1703 pa->pa_dmat = &dom->dmat;
1704}
1705
1706/* Create list of device scope entries from ACPI table */
1707void
1708acpidmar_parse_devscope(union acpidmar_entry *de, int off, int segment,
1709 struct devlist_head *devlist)
1710{
1711 struct acpidmar_devscope *ds;
1712 struct dmar_devlist *d;
1713 int dplen, i;
1714
1715 TAILQ_INIT(devlist)do { (devlist)->tqh_first = ((void *)0); (devlist)->tqh_last
= &(devlist)->tqh_first; } while (0);
1716 while (off < de->length) {
1717 ds = (struct acpidmar_devscope *)((unsigned char *)de + off);
1718 off += ds->length;
1719
1720 /* We only care about bridges and endpoints */
1721 if (ds->type != DMAR_ENDPOINT0x1 && ds->type != DMAR_BRIDGE0x2)
1722 continue;
1723
1724 dplen = ds->length - sizeof(*ds);
1725 d = malloc(sizeof(*d) + dplen, M_DEVBUF2, M_ZERO0x0008 | M_WAITOK0x0001);
1726 d->bus = ds->bus;
1727 d->type = ds->type;
1728 d->ndp = dplen / 2;
1729 d->dp = (void *)&d[1];
1730 memcpy(d->dp, &ds[1], dplen)__builtin_memcpy((d->dp), (&ds[1]), (dplen));
1731 TAILQ_INSERT_TAIL(devlist, d, link)do { (d)->link.tqe_next = ((void *)0); (d)->link.tqe_prev
= (devlist)->tqh_last; *(devlist)->tqh_last = (d); (devlist
)->tqh_last = &(d)->link.tqe_next; } while (0);
1732
1733 DPRINTF(1, " %8s %.4x:%.2x.%.2x.%x {",
1734 ds->type == DMAR_BRIDGE ? "bridge" : "endpoint",
1735 segment, ds->bus,
1736 d->dp[0].device,
1737 d->dp[0].function);
1738
1739 for (i = 1; i < d->ndp; i++) {
1740 DPRINTF(1, " %2x.%x ",
1741 d->dp[i].device,
1742 d->dp[i].function);
1743 }
1744 DPRINTF(1, "}\n");
1745 }
1746}
1747
1748/* DMA Remapping Hardware Unit */
1749void
1750acpidmar_drhd(struct acpidmar_softc *sc, union acpidmar_entry *de)
1751{
1752 struct iommu_softc *iommu;
1753
1754 printf("DRHD: segment:%.4x base:%.16llx flags:%.2x\n",
1755 de->drhd.segment,
1756 de->drhd.address,
1757 de->drhd.flags);
1758 iommu = malloc(sizeof(*iommu), M_DEVBUF2, M_ZERO0x0008 | M_WAITOK0x0001);
1759 acpidmar_parse_devscope(de, sizeof(de->drhd), de->drhd.segment,
1760 &iommu->devices);
1761 iommu_init(sc, iommu, &de->drhd);
1762
1763 if (de->drhd.flags) {
1764 /* Catchall IOMMU goes at end of list */
1765 TAILQ_INSERT_TAIL(&sc->sc_drhds, iommu, link)do { (iommu)->link.tqe_next = ((void *)0); (iommu)->link
.tqe_prev = (&sc->sc_drhds)->tqh_last; *(&sc->
sc_drhds)->tqh_last = (iommu); (&sc->sc_drhds)->
tqh_last = &(iommu)->link.tqe_next; } while (0);
1766 } else {
1767 TAILQ_INSERT_HEAD(&sc->sc_drhds, iommu, link)do { if (((iommu)->link.tqe_next = (&sc->sc_drhds)->
tqh_first) != ((void *)0)) (&sc->sc_drhds)->tqh_first
->link.tqe_prev = &(iommu)->link.tqe_next; else (&
sc->sc_drhds)->tqh_last = &(iommu)->link.tqe_next
; (&sc->sc_drhds)->tqh_first = (iommu); (iommu)->
link.tqe_prev = &(&sc->sc_drhds)->tqh_first; } while
(0);
1768 }
1769}
1770
1771/* Reserved Memory Region Reporting */
1772void
1773acpidmar_rmrr(struct acpidmar_softc *sc, union acpidmar_entry *de)
1774{
1775 struct rmrr_softc *rmrr;
1776 bios_memmap_t *im, *jm;
1777 uint64_t start, end;
1778
1779 printf("RMRR: segment:%.4x range:%.16llx-%.16llx\n",
1780 de->rmrr.segment, de->rmrr.base, de->rmrr.limit);
1781 if (de->rmrr.limit <= de->rmrr.base) {
1782 printf(" buggy BIOS\n");
1783 return;
1784 }
1785
1786 rmrr = malloc(sizeof(*rmrr), M_DEVBUF2, M_ZERO0x0008 | M_WAITOK0x0001);
1787 rmrr->start = trunc_page(de->rmrr.base)((de->rmrr.base) & ~((1 << 12) - 1));
1788 rmrr->end = round_page(de->rmrr.limit)(((de->rmrr.limit) + ((1 << 12) - 1)) & ~((1 <<
12) - 1));
1789 rmrr->segment = de->rmrr.segment;
1790 acpidmar_parse_devscope(de, sizeof(de->rmrr), de->rmrr.segment,
1791 &rmrr->devices);
1792
1793 for (im = bios_memmap; im->type != BIOS_MAP_END0x00; im++) {
1794 if (im->type != BIOS_MAP_RES0x02)
1795 continue;
1796 /* Search for adjacent reserved regions */
1797 start = im->addr;
1798 end = im->addr+im->size;
1799 for (jm = im+1; jm->type == BIOS_MAP_RES0x02 && end == jm->addr;
1800 jm++) {
1801 end = jm->addr+jm->size;
1802 }
1803 printf("e820: %.16llx - %.16llx\n", start, end);
1804 if (start <= rmrr->start && rmrr->end <= end) {
1805 /* Bah.. some buggy BIOS stomp outside RMRR */
1806 printf(" ** inside E820 Reserved %.16llx %.16llx\n",
1807 start, end);
1808 rmrr->start = trunc_page(start)((start) & ~((1 << 12) - 1));
1809 rmrr->end = round_page(end)(((end) + ((1 << 12) - 1)) & ~((1 << 12) - 1)
);
1810 break;
1811 }
1812 }
1813 TAILQ_INSERT_TAIL(&sc->sc_rmrrs, rmrr, link)do { (rmrr)->link.tqe_next = ((void *)0); (rmrr)->link.
tqe_prev = (&sc->sc_rmrrs)->tqh_last; *(&sc->
sc_rmrrs)->tqh_last = (rmrr); (&sc->sc_rmrrs)->tqh_last
= &(rmrr)->link.tqe_next; } while (0);
1814}
1815
1816/* Root Port ATS Reporting */
1817void
1818acpidmar_atsr(struct acpidmar_softc *sc, union acpidmar_entry *de)
1819{
1820 struct atsr_softc *atsr;
1821
1822 printf("ATSR: segment:%.4x flags:%x\n",
1823 de->atsr.segment,
1824 de->atsr.flags);
1825
1826 atsr = malloc(sizeof(*atsr), M_DEVBUF2, M_ZERO0x0008 | M_WAITOK0x0001);
1827 atsr->flags = de->atsr.flags;
1828 atsr->segment = de->atsr.segment;
1829 acpidmar_parse_devscope(de, sizeof(de->atsr), de->atsr.segment,
1830 &atsr->devices);
1831
1832 TAILQ_INSERT_TAIL(&sc->sc_atsrs, atsr, link)do { (atsr)->link.tqe_next = ((void *)0); (atsr)->link.
tqe_prev = (&sc->sc_atsrs)->tqh_last; *(&sc->
sc_atsrs)->tqh_last = (atsr); (&sc->sc_atsrs)->tqh_last
= &(atsr)->link.tqe_next; } while (0);
1833}
1834
1835void
1836acpidmar_init(struct acpidmar_softc *sc, struct acpi_dmar *dmar)
1837{
1838 struct rmrr_softc *rmrr;
1839 struct iommu_softc *iommu;
1840 struct domain *dom;
1841 struct dmar_devlist *dl;
1842 union acpidmar_entry *de;
1843 int off, sid, rc;
1844
1845 domain_map_page = domain_map_page_intel;
1846 printf(": hardware width: %d, intr_remap:%d x2apic_opt_out:%d\n",
1847 dmar->haw+1,
1848 !!(dmar->flags & 0x1),
1849 !!(dmar->flags & 0x2));
1850 sc->sc_haw = dmar->haw+1;
1851 sc->sc_flags = dmar->flags;
1852
1853 TAILQ_INIT(&sc->sc_drhds)do { (&sc->sc_drhds)->tqh_first = ((void *)0); (&
sc->sc_drhds)->tqh_last = &(&sc->sc_drhds)->
tqh_first; } while (0);
1854 TAILQ_INIT(&sc->sc_rmrrs)do { (&sc->sc_rmrrs)->tqh_first = ((void *)0); (&
sc->sc_rmrrs)->tqh_last = &(&sc->sc_rmrrs)->
tqh_first; } while (0);
1855 TAILQ_INIT(&sc->sc_atsrs)do { (&sc->sc_atsrs)->tqh_first = ((void *)0); (&
sc->sc_atsrs)->tqh_last = &(&sc->sc_atsrs)->
tqh_first; } while (0);
1856
1857 off = sizeof(*dmar);
1858 while (off < dmar->hdr.length) {
1859 de = (union acpidmar_entry *)((unsigned char *)dmar + off);
1860 switch (de->type) {
1861 case DMAR_DRHD0x0:
1862 acpidmar_drhd(sc, de);
1863 break;
1864 case DMAR_RMRR0x1:
1865 acpidmar_rmrr(sc, de);
1866 break;
1867 case DMAR_ATSR0x2:
1868 acpidmar_atsr(sc, de);
1869 break;
1870 default:
1871 printf("DMAR: unknown %x\n", de->type);
1872 break;
1873 }
1874 off += de->length;
1875 }
1876
1877 /* Pre-create domains for iommu devices */
1878 TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu)
!= ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
1879 TAILQ_FOREACH(dl, &iommu->devices, link)for((dl) = ((&iommu->devices)->tqh_first); (dl) != (
(void *)0); (dl) = ((dl)->link.tqe_next)) {
1880 sid = mksid(dl->bus, dl->dp[0].device,
1881 dl->dp[0].function);
1882 dom = acpidmar_pci_attach(sc, iommu->segment, sid, 0);
1883 if (dom != NULL((void *)0)) {
1884 printf("%.4x:%.2x:%.2x.%x iommu:%d did:%.4x\n",
1885 iommu->segment, dl->bus, dl->dp[0].device, dl->dp[0].function,
1886 iommu->id, dom->did);
1887 }
1888 }
1889 }
1890 /* Map passthrough pages for RMRR */
1891 TAILQ_FOREACH(rmrr, &sc->sc_rmrrs, link)for((rmrr) = ((&sc->sc_rmrrs)->tqh_first); (rmrr) !=
((void *)0); (rmrr) = ((rmrr)->link.tqe_next)) {
1892 TAILQ_FOREACH(dl, &rmrr->devices, link)for((dl) = ((&rmrr->devices)->tqh_first); (dl) != (
(void *)0); (dl) = ((dl)->link.tqe_next)) {
1893 sid = mksid(dl->bus, dl->dp[0].device,
1894 dl->dp[0].function);
1895 dom = acpidmar_pci_attach(sc, rmrr->segment, sid, 0);
1896 if (dom != NULL((void *)0)) {
1897 printf("%s map ident: %.16llx %.16llx\n",
1898 dom_bdf(dom), rmrr->start, rmrr->end);
1899 domain_map_pthru(dom, rmrr->start, rmrr->end);
1900 rc = extent_alloc_region(dom->iovamap,
1901 rmrr->start, rmrr->end,
1902 EX_WAITOK0x0001 | EX_CONFLICTOK0x0080);
1903 }
1904 }
1905 }
1906}
1907
1908
1909/*=====================================================
1910 * AMD Vi
1911 *=====================================================*/
1912void acpiivrs_ivhd(struct acpidmar_softc *, struct acpi_ivhd *);
1913int ivhd_iommu_init(struct acpidmar_softc *, struct iommu_softc *,
1914 struct acpi_ivhd *);
1915int _ivhd_issue_command(struct iommu_softc *, const struct ivhd_command *);
1916void ivhd_show_event(struct iommu_softc *, struct ivhd_event *evt, int);
1917int ivhd_issue_command(struct iommu_softc *, const struct ivhd_command *, int);
1918int ivhd_invalidate_domain(struct iommu_softc *, int);
1919void ivhd_intr_map(struct iommu_softc *, int);
1920void ivhd_checkerr(struct iommu_softc *iommu);
1921int acpiivhd_intr(void *);
1922
1923int
1924acpiivhd_intr(void *ctx)
1925{
1926 struct iommu_softc *iommu = ctx;
1927
1928 if (!iommu->dte)
1929 return (0);
1930 ivhd_poll_events(iommu);
1931 return (1);
1932}
1933
1934/* Setup interrupt for AMD */
1935void
1936ivhd_intr_map(struct iommu_softc *iommu, int devid) {
1937 pci_intr_handle_t ih;
1938
1939 if (iommu->intr)
1940 return;
1941 ih.tag = pci_make_tag(NULL((void *)0), sid_bus(devid), sid_dev(devid), sid_fun(devid));
1942 ih.line = APIC_INT_VIA_MSG0x20000000;
1943 ih.pin = 0;
1944 iommu->intr = pci_intr_establish(NULL((void *)0), ih, IPL_NET0x7 | IPL_MPSAFE0x100,
1945 acpiivhd_intr, iommu, "amd_iommu");
1946 printf("amd iommu intr: %p\n", iommu->intr);
1947}
1948
1949void
1950_dumppte(struct pte_entry *pte, int lvl, vaddr_t va)
1951{
1952 char *pfx[] = { " ", " ", " ", " ", "" };
1953 uint64_t i, sh;
1954 struct pte_entry *npte;
1955
1956 for (i = 0; i < 512; i++) {
1957 sh = (i << (((lvl-1) * 9) + 12));
1958 if (pte[i].val & PTE_P(1L << 0)) {
1959 if (lvl > 1) {
1960 npte = (void *)PMAP_DIRECT_MAP((pte[i].val & PTE_PADDR_MASK))((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + ((pte[i].val & 0x000FFFFFFFFFF000LL)));
1961 printf("%slvl%d: %.16llx nxt:%llu\n", pfx[lvl], lvl,
1962 pte[i].val, (pte[i].val >> 9) & 7);
1963 _dumppte(npte, lvl-1, va | sh);
1964 } else {
1965 printf("%slvl%d: %.16llx <- %.16llx \n", pfx[lvl], lvl,
1966 pte[i].val, va | sh);
1967 }
1968 }
1969 }
1970}
1971
1972void
1973ivhd_showpage(struct iommu_softc *iommu, int sid, paddr_t paddr)
1974{
1975 struct domain *dom;
1976 static int show = 0;
1977
1978 if (show > 10)
1979 return;
1980 show++;
1981 dom = acpidmar_pci_attach(acpidmar_sc, 0, sid, 0);
1982 if (!dom)
1983 return;
1984 printf("DTE: %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
1985 iommu->dte[sid].dw0,
1986 iommu->dte[sid].dw1,
1987 iommu->dte[sid].dw2,
1988 iommu->dte[sid].dw3,
1989 iommu->dte[sid].dw4,
1990 iommu->dte[sid].dw5,
1991 iommu->dte[sid].dw6,
1992 iommu->dte[sid].dw7);
1993 _dumppte(dom->pte, 3, 0);
1994}
1995
1996/* Display AMD IOMMU Error */
1997void
1998ivhd_show_event(struct iommu_softc *iommu, struct ivhd_event *evt, int head)
1999{
2000 int type, sid, did, flag;
2001 uint64_t address;
2002
2003 /* Get Device, Domain, Address and Type of event */
2004 sid = __EXTRACT(evt->dw0, EVT_SID)(((evt->dw0) >> 0) & 0xFFFF);
2005 type = __EXTRACT(evt->dw1, EVT_TYPE)(((evt->dw1) >> 28) & 0xF);
2006 did = __EXTRACT(evt->dw1, EVT_DID)(((evt->dw1) >> 0) & 0xFFFF);
2007 flag = __EXTRACT(evt->dw1, EVT_FLAG)(((evt->dw1) >> 16) & 0xFFF);
2008 address = _get64(&evt->dw2)*(uint64_t *)(&evt->dw2);
2009
2010 printf("=== IOMMU Error[%.4x]: ", head);
2011 switch (type) {
2012 case ILLEGAL_DEV_TABLE_ENTRY:
2013 printf("illegal dev table entry dev=%s addr=0x%.16llx %s, %s, %s, %s\n",
2014 dmar_bdf(sid), address,
2015 evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2016 evt->dw1 & EVT_RZ(1L << 23) ? "reserved bit" : "invalid level",
2017 evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2018 evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2019 ivhd_showdte(iommu);
2020 break;
2021 case IO_PAGE_FAULT:
2022 printf("io page fault dev=%s did=0x%.4x addr=0x%.16llx\n%s, %s, %s, %s, %s, %s\n",
2023 dmar_bdf(sid), did, address,
2024 evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2025 evt->dw1 & EVT_RZ(1L << 23) ? "reserved bit" : "invalid level",
2026 evt->dw1 & EVT_PE(1L << 22) ? "no perm" : "perm",
2027 evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2028 evt->dw1 & EVT_PR(1L << 20) ? "present" : "not present",
2029 evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2030 ivhd_showdte(iommu);
2031 ivhd_showpage(iommu, sid, address);
2032 break;
2033 case DEV_TAB_HARDWARE_ERROR:
2034 printf("device table hardware error dev=%s addr=0x%.16llx %s, %s, %s\n",
2035 dmar_bdf(sid), address,
2036 evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2037 evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2038 evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2039 ivhd_showdte(iommu);
2040 break;
2041 case PAGE_TAB_HARDWARE_ERROR:
2042 printf("page table hardware error dev=%s addr=0x%.16llx %s, %s, %s\n",
2043 dmar_bdf(sid), address,
2044 evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2045 evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2046 evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2047 ivhd_showdte(iommu);
2048 break;
2049 case ILLEGAL_COMMAND_ERROR:
2050 printf("illegal command addr=0x%.16llx\n", address);
2051 ivhd_showcmd(iommu);
2052 break;
2053 case COMMAND_HARDWARE_ERROR:
2054 printf("command hardware error addr=0x%.16llx flag=0x%.4x\n",
2055 address, flag);
2056 ivhd_showcmd(iommu);
2057 break;
2058 case IOTLB_INV_TIMEOUT:
2059 printf("iotlb invalidation timeout dev=%s address=0x%.16llx\n",
2060 dmar_bdf(sid), address);
2061 break;
2062 case INVALID_DEVICE_REQUEST:
2063 printf("invalid device request dev=%s addr=0x%.16llx flag=0x%.4x\n",
2064 dmar_bdf(sid), address, flag);
2065 break;
2066 default:
2067 printf("unknown type=0x%.2x\n", type);
2068 break;
2069 }
2070 /* Clear old event */
2071 evt->dw0 = 0;
2072 evt->dw1 = 0;
2073 evt->dw2 = 0;
2074 evt->dw3 = 0;
2075}
2076
2077/* AMD: Process IOMMU error from hardware */
2078int
2079ivhd_poll_events(struct iommu_softc *iommu)
2080{
2081 uint32_t head, tail;
2082 int sz;
2083
2084 sz = sizeof(struct ivhd_event);
2085 head = iommu_read_4(iommu, EVT_HEAD_REG0x2010);
2086 tail = iommu_read_4(iommu, EVT_TAIL_REG0x2018);
2087 if (head == tail) {
2088 /* No pending events */
2089 return (0);
2090 }
2091 while (head != tail) {
2092 ivhd_show_event(iommu, iommu->evt_tbl + head, head);
2093 head = (head + sz) % EVT_TBL_SIZE4096;
2094 }
2095 iommu_write_4(iommu, EVT_HEAD_REG0x2010, head);
2096 return (0);
2097}
2098
2099/* AMD: Issue command to IOMMU queue */
2100int
2101_ivhd_issue_command(struct iommu_softc *iommu, const struct ivhd_command *cmd)
2102{
2103 u_long rf;
2104 uint32_t head, tail, next;
2105 int sz;
2106
2107 head = iommu_read_4(iommu, CMD_HEAD_REG0x2000);
2108 sz = sizeof(*cmd);
2109 rf = intr_disable();
2110 tail = iommu_read_4(iommu, CMD_TAIL_REG0x2008);
2111 next = (tail + sz) % CMD_TBL_SIZE4096;
2112 if (next == head) {
2113 printf("FULL\n");
2114 /* Queue is full */
2115 intr_restore(rf);
2116 return -EBUSY16;
2117 }
2118 memcpy(iommu->cmd_tbl + tail, cmd, sz)__builtin_memcpy((iommu->cmd_tbl + tail), (cmd), (sz));
2119 iommu_write_4(iommu, CMD_TAIL_REG0x2008, next);
2120 intr_restore(rf);
2121 return (tail / sz);
2122}
2123
2124#define IVHD_MAXDELAY8 8
2125
2126int
2127ivhd_issue_command(struct iommu_softc *iommu, const struct ivhd_command *cmd, int wait)
2128{
2129 struct ivhd_command wq = { 0 };
2130 volatile uint64_t wv __aligned(16)__attribute__((__aligned__(16))) = 0LL;
2131 paddr_t paddr;
2132 int rc, i;
2133
2134 rc = _ivhd_issue_command(iommu, cmd);
2135 if (rc >= 0 && wait) {
2136 /* Wait for previous commands to complete.
2137 * Store address of completion variable to command */
2138 pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)&wv, &paddr);
2139 wq.dw0 = (paddr & ~0xF) | 0x1;
2140 wq.dw1 = (COMPLETION_WAIT << CMD_SHIFT28) | ((paddr >> 32) & 0xFFFFF);
2141 wq.dw2 = 0xDEADBEEF;
2142 wq.dw3 = 0xFEEDC0DE;
2143
2144 rc = _ivhd_issue_command(iommu, &wq);
2145 /* wv will change to value in dw2/dw3 when command is complete */
2146 for (i = 0; i < IVHD_MAXDELAY8 && !wv; i++) {
2147 DELAY(10 << i)(*delay_func)(10 << i);
2148 }
2149 if (i == IVHD_MAXDELAY8) {
2150 printf("ivhd command timeout: %.8x %.8x %.8x %.8x wv:%llx idx:%x\n",
2151 cmd->dw0, cmd->dw1, cmd->dw2, cmd->dw3, wv, rc);
2152 }
2153 }
2154 return rc;
2155
2156}
2157
2158/* AMD: Flush changes to Device Table Entry for a specific domain */
2159int
2160ivhd_flush_devtab(struct iommu_softc *iommu, int did)
2161{
2162 struct ivhd_command cmd = {
2163 .dw0 = did,
2164 .dw1 = INVALIDATE_DEVTAB_ENTRY << CMD_SHIFT28
2165 };
2166
2167 return ivhd_issue_command(iommu, &cmd, 1);
2168}
2169
2170/* AMD: Invalidate all IOMMU device and page tables */
2171int
2172ivhd_invalidate_iommu_all(struct iommu_softc *iommu)
2173{
2174 struct ivhd_command cmd = {
2175 .dw1 = INVALIDATE_IOMMU_ALL << CMD_SHIFT28
2176 };
2177
2178 return ivhd_issue_command(iommu, &cmd, 0);
2179}
2180
2181/* AMD: Invalidate interrupt remapping */
2182int
2183ivhd_invalidate_interrupt_table(struct iommu_softc *iommu, int did)
2184{
2185 struct ivhd_command cmd = {
2186 .dw0 = did,
2187 .dw1 = INVALIDATE_INTERRUPT_TABLE << CMD_SHIFT28
2188 };
2189
2190 return ivhd_issue_command(iommu, &cmd, 0);
2191}
2192
2193/* AMD: Invalidate all page tables in a domain */
2194int
2195ivhd_invalidate_domain(struct iommu_softc *iommu, int did)
2196{
2197 struct ivhd_command cmd = { .dw1 = did | (INVALIDATE_IOMMU_PAGES << CMD_SHIFT28) };
2198
2199 cmd.dw2 = 0xFFFFF000 | 0x3;
2200 cmd.dw3 = 0x7FFFFFFF;
2201 return ivhd_issue_command(iommu, &cmd, 1);
2202}
2203
2204/* AMD: Display Registers */
2205void
2206ivhd_showreg(struct iommu_softc *iommu)
2207{
2208 printf("---- dt:%.16llx cmd:%.16llx evt:%.16llx ctl:%.16llx sts:%.16llx\n",
2209 iommu_read_8(iommu, DEV_TAB_BASE_REG0x0000),
2210 iommu_read_8(iommu, CMD_BASE_REG0x0008),
2211 iommu_read_8(iommu, EVT_BASE_REG0x0010),
2212 iommu_read_8(iommu, IOMMUCTL_REG0x0018),
2213 iommu_read_8(iommu, IOMMUSTS_REG0x2020));
2214 printf("---- cmd queue:%.16llx %.16llx evt queue:%.16llx %.16llx\n",
2215 iommu_read_8(iommu, CMD_HEAD_REG0x2000),
2216 iommu_read_8(iommu, CMD_TAIL_REG0x2008),
2217 iommu_read_8(iommu, EVT_HEAD_REG0x2010),
2218 iommu_read_8(iommu, EVT_TAIL_REG0x2018));
2219}
2220
2221/* AMD: Generate Errors to test event handler */
2222void
2223ivhd_checkerr(struct iommu_softc *iommu)
2224{
2225 struct ivhd_command cmd = { -1, -1, -1, -1 };
2226
2227 /* Generate ILLEGAL DEV TAB entry? */
2228 iommu->dte[0x2303].dw0 = -1; /* invalid */
2229 iommu->dte[0x2303].dw2 = 0x1234; /* domain */
2230 iommu->dte[0x2303].dw7 = -1; /* reserved */
2231 ivhd_flush_devtab(iommu, 0x1234);
2232 ivhd_poll_events(iommu);
2233
2234 /* Generate ILLEGAL_COMMAND_ERROR : ok */
2235 ivhd_issue_command(iommu, &cmd, 0);
2236 ivhd_poll_events(iommu);
2237
2238 /* Generate page hardware error */
2239}
2240
2241/* AMD: Show Device Table Entry */
2242void
2243ivhd_showdte(struct iommu_softc *iommu)
2244{
2245 int i;
2246
2247 for (i = 0; i < 65536; i++) {
2248 if (iommu->dte[i].dw0) {
2249 printf("%.2x:%.2x.%x: %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
2250 i >> 8, (i >> 3) & 0x1F, i & 0x7,
2251 iommu->dte[i].dw0, iommu->dte[i].dw1,
2252 iommu->dte[i].dw2, iommu->dte[i].dw3,
2253 iommu->dte[i].dw4, iommu->dte[i].dw5,
2254 iommu->dte[i].dw6, iommu->dte[i].dw7);
2255 }
2256 }
2257}
2258
2259/* AMD: Show command entries */
2260void
2261ivhd_showcmd(struct iommu_softc *iommu)
2262{
2263 struct ivhd_command *ihd;
2264 paddr_t phd;
2265 int i;
2266
2267 ihd = iommu->cmd_tbl;
2268 phd = iommu_read_8(iommu, CMD_BASE_REG0x0008) & CMD_BASE_MASK0x000FFFFFFFFFF000LL;
2269 for (i = 0; i < 4096 / 128; i++) {
2270 printf("%.2x: %.16llx %.8x %.8x %.8x %.8x\n", i,
2271 (uint64_t)phd + i * sizeof(*ihd),
2272 ihd[i].dw0,ihd[i].dw1,ihd[i].dw2,ihd[i].dw3);
2273 }
2274}
2275
2276#define _c(x)(int)((iommu->ecap >> x_SHIFT) & x_MASK) (int)((iommu->ecap >> x ##_SHIFT) & x ## _MASK)
2277
2278/* AMD: Initialize IOMMU */
2279int
2280ivhd_iommu_init(struct acpidmar_softc *sc, struct iommu_softc *iommu,
2281 struct acpi_ivhd *ivhd)
2282{
2283 static int niommu;
2284 paddr_t paddr;
2285 uint64_t ov;
2286
2287 if (sc == NULL((void *)0) || iommu == NULL((void *)0) || ivhd == NULL((void *)0)) {
2288 printf("Bad pointer to iommu_init!\n");
2289 return -1;
2290 }
2291 if (_bus_space_map(sc->sc_memt, ivhd->address, 0x80000, 0, &iommu->ioh) != 0) {
2292 printf("Bus Space Map fails\n");
2293 return -1;
2294 }
2295 TAILQ_INIT(&iommu->domains)do { (&iommu->domains)->tqh_first = ((void *)0); (&
iommu->domains)->tqh_last = &(&iommu->domains
)->tqh_first; } while (0);
2296 TAILQ_INIT(&iommu->devices)do { (&iommu->devices)->tqh_first = ((void *)0); (&
iommu->devices)->tqh_last = &(&iommu->devices
)->tqh_first; } while (0);
2297
2298 /* Setup address width and number of domains */
2299 iommu->id = ++niommu;
2300 iommu->iot = sc->sc_memt;
2301 iommu->mgaw = 48;
2302 iommu->agaw = 48;
2303 iommu->flags = 1;
2304 iommu->segment = 0;
2305 iommu->ndoms = 256;
2306
2307 printf(": AMD iommu%d at 0x%.8llx\n", iommu->id, ivhd->address);
2308
2309 iommu->ecap = iommu_read_8(iommu, EXTFEAT_REG0x0030);
2310 DPRINTF(0,"iommu%d: ecap:%.16llx ", iommu->id, iommu->ecap);
2311 DPRINTF(0,"%s%s%s%s%s%s%s%s\n",
2312 iommu->ecap & EFR_PREFSUP ? "pref " : "",
2313 iommu->ecap & EFR_PPRSUP ? "ppr " : "",
2314 iommu->ecap & EFR_NXSUP ? "nx " : "",
2315 iommu->ecap & EFR_GTSUP ? "gt " : "",
2316 iommu->ecap & EFR_IASUP ? "ia " : "",
2317 iommu->ecap & EFR_GASUP ? "ga " : "",
2318 iommu->ecap & EFR_HESUP ? "he " : "",
2319 iommu->ecap & EFR_PCSUP ? "pc " : "");
2320 DPRINTF(0,"hats:%x gats:%x glxsup:%x smif:%x smifrc:%x gam:%x\n",
2321 _c(EFR_HATS), _c(EFR_GATS), _c(EFR_GLXSUP), _c(EFR_SMIFSUP),
2322 _c(EFR_SMIFRC), _c(EFR_GAMSUP));
2323
2324 /* Turn off iommu */
2325 ov = iommu_read_8(iommu, IOMMUCTL_REG0x0018);
2326 iommu_write_8(iommu, IOMMUCTL_REG0x0018, ov & ~(CTL_IOMMUEN(1L << 0) | CTL_COHERENT(1L << 10) |
2327 CTL_HTTUNEN(1L << 1) | CTL_RESPASSPW(1L << 9) | CTL_PASSPW(1L << 8) | CTL_ISOC(1L << 11)));
2328
2329 /* Enable intr, mark IOMMU device as invalid for remap */
2330 sid_flag[ivhd->devid] |= SID_INVALID0x80000000L;
2331 ivhd_intr_map(iommu, ivhd->devid);
2332
2333 /* Setup command buffer with 4k buffer (128 entries) */
2334 iommu->cmd_tbl = iommu_alloc_page(iommu, &paddr);
2335 iommu_write_8(iommu, CMD_BASE_REG0x0008, (paddr & CMD_BASE_MASK0x000FFFFFFFFFF000LL) | CMD_TBL_LEN_4K(8LL << 56));
2336 iommu_write_4(iommu, CMD_HEAD_REG0x2000, 0x00);
2337 iommu_write_4(iommu, CMD_TAIL_REG0x2008, 0x00);
2338 iommu->cmd_tblp = paddr;
2339
2340 /* Setup event log with 4k buffer (128 entries) */
2341 iommu->evt_tbl = iommu_alloc_page(iommu, &paddr);
2342 iommu_write_8(iommu, EVT_BASE_REG0x0010, (paddr & EVT_BASE_MASK0x000FFFFFFFFFF000LL) | EVT_TBL_LEN_4K(8LL << 56));
2343 iommu_write_4(iommu, EVT_HEAD_REG0x2010, 0x00);
2344 iommu_write_4(iommu, EVT_TAIL_REG0x2018, 0x00);
2345 iommu->evt_tblp = paddr;
2346
2347 /* Setup device table
2348 * 1 entry per source ID (bus:device:function - 64k entries)
2349 */
2350 iommu->dte = sc->sc_hwdte;
2351 iommu_write_8(iommu, DEV_TAB_BASE_REG0x0000, (sc->sc_hwdtep & DEV_TAB_MASK0x000FFFFFFFFFF000LL) | DEV_TAB_LEN0x1FF);
2352
2353 /* Enable IOMMU */
2354 ov |= (CTL_IOMMUEN(1L << 0) | CTL_EVENTLOGEN(1L << 2) | CTL_CMDBUFEN(1L << 12) | CTL_EVENTINTEN(1L << 3));
2355 if (ivhd->flags & IVHD_COHERENT(1L << 5))
2356 ov |= CTL_COHERENT(1L << 10);
2357 if (ivhd->flags & IVHD_HTTUNEN(1L << 0))
2358 ov |= CTL_HTTUNEN(1L << 1);
2359 if (ivhd->flags & IVHD_RESPASSPW(1L << 2))
2360 ov |= CTL_RESPASSPW(1L << 9);
2361 if (ivhd->flags & IVHD_PASSPW(1L << 1))
2362 ov |= CTL_PASSPW(1L << 8);
2363 if (ivhd->flags & IVHD_ISOC(1L << 3))
2364 ov |= CTL_ISOC(1L << 11);
2365 ov &= ~(CTL_INVTIMEOUT_MASK0x7 << CTL_INVTIMEOUT_SHIFT5);
2366 ov |= (CTL_INVTIMEOUT_10MS2 << CTL_INVTIMEOUT_SHIFT5);
2367 iommu_write_8(iommu, IOMMUCTL_REG0x0018, ov);
2368
2369 ivhd_invalidate_iommu_all(iommu);
2370
2371 TAILQ_INSERT_TAIL(&sc->sc_drhds, iommu, link)do { (iommu)->link.tqe_next = ((void *)0); (iommu)->link
.tqe_prev = (&sc->sc_drhds)->tqh_last; *(&sc->
sc_drhds)->tqh_last = (iommu); (&sc->sc_drhds)->
tqh_last = &(iommu)->link.tqe_next; } while (0);
2372 return 0;
2373}
2374
2375void
2376acpiivrs_ivhd(struct acpidmar_softc *sc, struct acpi_ivhd *ivhd)
2377{
2378 struct iommu_softc *iommu;
2379 struct acpi_ivhd_ext *ext;
2380 union acpi_ivhd_entry *ie;
2381 int start, off, dte, all_dte = 0;
2382
2383 if (ivhd->type == IVRS_IVHD_EXT0x11) {
2384 ext = (struct acpi_ivhd_ext *)ivhd;
2385 DPRINTF(0,"ivhd: %.2x %.2x %.4x %.4x:%s %.4x %.16llx %.4x %.8x %.16llx\n",
2386 ext->type, ext->flags, ext->length,
2387 ext->segment, dmar_bdf(ext->devid), ext->cap,
2388 ext->address, ext->info,
2389 ext->attrib, ext->efr);
2390 if (ext->flags & IVHD_PPRSUP(1L << 7))
2391 DPRINTF(0," PPRSup");
2392 if (ext->flags & IVHD_PREFSUP(1L << 6))
2393 DPRINTF(0," PreFSup");
2394 if (ext->flags & IVHD_COHERENT(1L << 5))
2395 DPRINTF(0," Coherent");
2396 if (ext->flags & IVHD_IOTLB(1L << 4))
2397 DPRINTF(0," Iotlb");
2398 if (ext->flags & IVHD_ISOC(1L << 3))
2399 DPRINTF(0," ISoc");
2400 if (ext->flags & IVHD_RESPASSPW(1L << 2))
2401 DPRINTF(0," ResPassPW");
2402 if (ext->flags & IVHD_PASSPW(1L << 1))
2403 DPRINTF(0," PassPW");
2404 if (ext->flags & IVHD_HTTUNEN(1L << 0))
2405 DPRINTF(0, " HtTunEn");
2406 if (ext->flags)
2407 DPRINTF(0,"\n");
2408 off = sizeof(*ext);
2409 iommu = malloc(sizeof(*iommu), M_DEVBUF2, M_ZERO0x0008|M_WAITOK0x0001);
2410 ivhd_iommu_init(sc, iommu, ivhd);
2411 } else {
2412 DPRINTF(0,"ivhd: %.2x %.2x %.4x %.4x:%s %.4x %.16llx %.4x %.8x\n",
2413 ivhd->type, ivhd->flags, ivhd->length,
2414 ivhd->segment, dmar_bdf(ivhd->devid), ivhd->cap,
2415 ivhd->address, ivhd->info,
2416 ivhd->feature);
2417 if (ivhd->flags & IVHD_PPRSUP(1L << 7))
2418 DPRINTF(0," PPRSup");
2419 if (ivhd->flags & IVHD_PREFSUP(1L << 6))
2420 DPRINTF(0," PreFSup");
2421 if (ivhd->flags & IVHD_COHERENT(1L << 5))
2422 DPRINTF(0," Coherent");
2423 if (ivhd->flags & IVHD_IOTLB(1L << 4))
2424 DPRINTF(0," Iotlb");
2425 if (ivhd->flags & IVHD_ISOC(1L << 3))
2426 DPRINTF(0," ISoc");
2427 if (ivhd->flags & IVHD_RESPASSPW(1L << 2))
2428 DPRINTF(0," ResPassPW");
2429 if (ivhd->flags & IVHD_PASSPW(1L << 1))
2430 DPRINTF(0," PassPW");
2431 if (ivhd->flags & IVHD_HTTUNEN(1L << 0))
2432 DPRINTF(0, " HtTunEn");
2433 if (ivhd->flags)
2434 DPRINTF(0,"\n");
2435 off = sizeof(*ivhd);
2436 }
2437 while (off < ivhd->length) {
2438 ie = (void *)ivhd + off;
2439 switch (ie->type) {
2440 case IVHD_ALL1:
2441 all_dte = ie->all.data;
2442 DPRINTF(0," ALL %.4x\n", dte);
2443 off += sizeof(ie->all);
2444 break;
2445 case IVHD_SEL2:
2446 dte = ie->sel.data;
2447 DPRINTF(0," SELECT: %s %.4x\n", dmar_bdf(ie->sel.devid), dte);
2448 off += sizeof(ie->sel);
2449 break;
2450 case IVHD_SOR3:
2451 dte = ie->sor.data;
2452 start = ie->sor.devid;
2453 DPRINTF(0," SOR: %s %.4x\n", dmar_bdf(start), dte);
2454 off += sizeof(ie->sor);
2455 break;
2456 case IVHD_EOR4:
2457 DPRINTF(0," EOR: %s\n", dmar_bdf(ie->eor.devid));
2458 off += sizeof(ie->eor);
2459 break;
2460 case IVHD_ALIAS_SEL66:
2461 dte = ie->alias.data;
2462 DPRINTF(0," ALIAS: src=%s: ", dmar_bdf(ie->alias.srcid));
2463 DPRINTF(0," %s %.4x\n", dmar_bdf(ie->alias.devid), dte);
2464 off += sizeof(ie->alias);
2465 break;
2466 case IVHD_ALIAS_SOR67:
2467 dte = ie->alias.data;
2468 DPRINTF(0," ALIAS_SOR: %s %.4x ", dmar_bdf(ie->alias.devid), dte);
2469 DPRINTF(0," src=%s\n", dmar_bdf(ie->alias.srcid));
2470 off += sizeof(ie->alias);
2471 break;
2472 case IVHD_EXT_SEL70:
2473 dte = ie->ext.data;
2474 DPRINTF(0," EXT SEL: %s %.4x %.8x\n", dmar_bdf(ie->ext.devid),
2475 dte, ie->ext.extdata);
2476 off += sizeof(ie->ext);
2477 break;
2478 case IVHD_EXT_SOR71:
2479 dte = ie->ext.data;
2480 DPRINTF(0," EXT SOR: %s %.4x %.8x\n", dmar_bdf(ie->ext.devid),
2481 dte, ie->ext.extdata);
2482 off += sizeof(ie->ext);
2483 break;
2484 case IVHD_SPECIAL72:
2485 DPRINTF(0," SPECIAL\n");
2486 off += sizeof(ie->special);
2487 break;
2488 default:
2489 DPRINTF(0," 2:unknown %x\n", ie->type);
2490 off = ivhd->length;
2491 break;
2492 }
2493 }
2494}
2495
2496void
2497acpiivrs_init(struct acpidmar_softc *sc, struct acpi_ivrs *ivrs)
2498{
2499 union acpi_ivrs_entry *ie;
2500 int off;
2501
2502 if (!sc->sc_hwdte) {
2503 sc->sc_hwdte = iommu_alloc_hwdte(sc, HWDTE_SIZE(65536 * sizeof(struct ivhd_dte)), &sc->sc_hwdtep);
2504 if (sc->sc_hwdte == NULL((void *)0))
2505 panic("Can't allocate HWDTE!");
2506 }
2507
2508 domain_map_page = domain_map_page_amd;
2509 DPRINTF(0,"IVRS Version: %d\n", ivrs->hdr.revision);
2510 DPRINTF(0," VA Size: %d\n",
2511 (ivrs->ivinfo >> IVRS_VASIZE_SHIFT) & IVRS_VASIZE_MASK);
2512 DPRINTF(0," PA Size: %d\n",
2513 (ivrs->ivinfo >> IVRS_PASIZE_SHIFT) & IVRS_PASIZE_MASK);
2514
2515 TAILQ_INIT(&sc->sc_drhds)do { (&sc->sc_drhds)->tqh_first = ((void *)0); (&
sc->sc_drhds)->tqh_last = &(&sc->sc_drhds)->
tqh_first; } while (0);
2516 TAILQ_INIT(&sc->sc_rmrrs)do { (&sc->sc_rmrrs)->tqh_first = ((void *)0); (&
sc->sc_rmrrs)->tqh_last = &(&sc->sc_rmrrs)->
tqh_first; } while (0);
2517 TAILQ_INIT(&sc->sc_atsrs)do { (&sc->sc_atsrs)->tqh_first = ((void *)0); (&
sc->sc_atsrs)->tqh_last = &(&sc->sc_atsrs)->
tqh_first; } while (0);
2518
2519 DPRINTF(0,"======== IVRS\n");
2520 off = sizeof(*ivrs);
2521 while (off < ivrs->hdr.length) {
2522 ie = (void *)ivrs + off;
2523 switch (ie->type) {
2524 case IVRS_IVHD0x10:
2525 case IVRS_IVHD_EXT0x11:
2526 acpiivrs_ivhd(sc, &ie->ivhd);
2527 break;
2528 case IVRS_IVMD_ALL0x20:
2529 case IVRS_IVMD_SPECIFIED0x21:
2530 case IVRS_IVMD_RANGE0x22:
2531 DPRINTF(0,"ivmd\n");
2532 break;
2533 default:
2534 DPRINTF(0,"1:unknown: %x\n", ie->type);
2535 break;
2536 }
2537 off += ie->length;
2538 }
2539 DPRINTF(0,"======== End IVRS\n");
2540}
2541
2542static int
2543acpiivhd_activate(struct iommu_softc *iommu, int act)
2544{
2545 switch (act) {
2546 case DVACT_SUSPEND3:
2547 iommu->flags |= IOMMU_FLAGS_SUSPEND0x4;
2548 break;
2549 case DVACT_RESUME4:
2550 iommu->flags &= ~IOMMU_FLAGS_SUSPEND0x4;
2551 break;
2552 }
2553 return (0);
2554}
2555
2556int
2557acpidmar_activate(struct device *self, int act)
2558{
2559 struct acpidmar_softc *sc = (struct acpidmar_softc *)self;
2560 struct iommu_softc *iommu;
2561
2562 printf("called acpidmar_activate %d %p\n", act, sc);
2563
2564 if (sc == NULL((void *)0)) {
2565 return (0);
2566 }
2567
2568 switch (act) {
2569 case DVACT_RESUME4:
2570 TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu)
!= ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
2571 printf("iommu%d resume\n", iommu->id);
2572 if (iommu->dte) {
2573 acpiivhd_activate(iommu, act);
2574 continue;
2575 }
2576 iommu_flush_write_buffer(iommu);
2577 iommu_set_rtaddr(iommu, iommu->rtaddr);
2578 iommu_write_4(iommu, DMAR_FEDATA_REG0x3c, iommu->fedata);
2579 iommu_write_4(iommu, DMAR_FEADDR_REG0x40, iommu->feaddr);
2580 iommu_write_4(iommu, DMAR_FEUADDR_REG0x44,
2581 iommu->feaddr >> 32);
2582 if ((iommu->flags & (IOMMU_FLAGS_BAD0x2|IOMMU_FLAGS_SUSPEND0x4)) ==
2583 IOMMU_FLAGS_SUSPEND0x4) {
2584 printf("enable wakeup translation\n");
2585 iommu_enable_translation(iommu, 1);
2586 }
2587 iommu_showcfg(iommu, -1);
2588 }
2589 break;
2590 case DVACT_SUSPEND3:
2591 TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu)
!= ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
2592 printf("iommu%d suspend\n", iommu->id);
2593 if (iommu->flags & IOMMU_FLAGS_BAD0x2)
2594 continue;
2595 if (iommu->dte) {
2596 acpiivhd_activate(iommu, act);
2597 continue;
2598 }
2599 iommu->flags |= IOMMU_FLAGS_SUSPEND0x4;
2600 iommu_enable_translation(iommu, 0);
2601 iommu_showcfg(iommu, -1);
2602 }
2603 break;
2604 }
2605 return (0);
2606}
2607
2608int
2609acpidmar_match(struct device *parent, void *match, void *aux)
2610{
2611 struct acpi_attach_args *aaa = aux;
2612 struct acpi_table_header *hdr;
2613
2614 /* If we do not have a table, it is not us */
2615 if (aaa->aaa_table == NULL((void *)0))
2616 return (0);
2617
2618 /* If it is an DMAR table, we can attach */
2619 hdr = (struct acpi_table_header *)aaa->aaa_table;
2620 if (memcmp(hdr->signature, DMAR_SIG, sizeof(DMAR_SIG) - 1)__builtin_memcmp((hdr->signature), ("DMAR"), (sizeof("DMAR"
) - 1)) == 0)
2621 return (1);
2622 if (memcmp(hdr->signature, IVRS_SIG, sizeof(IVRS_SIG) - 1)__builtin_memcmp((hdr->signature), ("IVRS"), (sizeof("IVRS"
) - 1)) == 0)
2623 return (1);
2624
2625 return (0);
2626}
2627
2628void
2629acpidmar_attach(struct device *parent, struct device *self, void *aux)
2630{
2631 struct acpidmar_softc *sc = (void *)self;
2632 struct acpi_attach_args *aaa = aux;
2633 struct acpi_dmar *dmar = (struct acpi_dmar *)aaa->aaa_table;
2634 struct acpi_ivrs *ivrs = (struct acpi_ivrs *)aaa->aaa_table;
2635 struct acpi_table_header *hdr;
2636
2637 hdr = (struct acpi_table_header *)aaa->aaa_table;
2638 sc->sc_memt = aaa->aaa_memt;
2639 sc->sc_dmat = aaa->aaa_dmat;
2640 if (memcmp(hdr->signature, DMAR_SIG, sizeof(DMAR_SIG) - 1)__builtin_memcmp((hdr->signature), ("DMAR"), (sizeof("DMAR"
) - 1)) == 0) {
2641 acpidmar_sc = sc;
2642 acpidmar_init(sc, dmar);
2643 }
2644 if (memcmp(hdr->signature, IVRS_SIG, sizeof(IVRS_SIG) - 1)__builtin_memcmp((hdr->signature), ("IVRS"), (sizeof("IVRS"
) - 1)) == 0) {
2645 acpidmar_sc = sc;
2646 acpiivrs_init(sc, ivrs);
2647 }
2648}
2649
2650/* Interrupt shiz */
2651void acpidmar_msi_hwmask(struct pic *, int);
2652void acpidmar_msi_hwunmask(struct pic *, int);
2653void acpidmar_msi_addroute(struct pic *, struct cpu_info *, int, int, int);
2654void acpidmar_msi_delroute(struct pic *, struct cpu_info *, int, int, int);
2655
2656void
2657acpidmar_msi_hwmask(struct pic *pic, int pin)
2658{
2659 struct iommu_pic *ip = (void *)pic;
2660 struct iommu_softc *iommu = ip->iommu;
2661
2662 printf("msi_hwmask\n");
2663
2664 mtx_enter(&iommu->reg_lock);
2665
2666 iommu_write_4(iommu, DMAR_FECTL_REG0x38, FECTL_IM(1LL << 31));
2667 iommu_read_4(iommu, DMAR_FECTL_REG0x38);
2668
2669 mtx_leave(&iommu->reg_lock);
2670}
2671
2672void
2673acpidmar_msi_hwunmask(struct pic *pic, int pin)
2674{
2675 struct iommu_pic *ip = (void *)pic;
2676 struct iommu_softc *iommu = ip->iommu;
2677
2678 printf("msi_hwunmask\n");
2679
2680 mtx_enter(&iommu->reg_lock);
2681
2682 iommu_write_4(iommu, DMAR_FECTL_REG0x38, 0);
2683 iommu_read_4(iommu, DMAR_FECTL_REG0x38);
2684
2685 mtx_leave(&iommu->reg_lock);
2686}
2687
2688void
2689acpidmar_msi_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec,
2690 int type)
2691{
2692 struct iommu_pic *ip = (void *)pic;
2693 struct iommu_softc *iommu = ip->iommu;
2694
2695 mtx_enter(&iommu->reg_lock);
2696
2697 iommu->fedata = vec;
2698 iommu->feaddr = 0xfee00000L | (ci->ci_apicid << 12);
2699 iommu_write_4(iommu, DMAR_FEDATA_REG0x3c, vec);
2700 iommu_write_4(iommu, DMAR_FEADDR_REG0x40, iommu->feaddr);
2701 iommu_write_4(iommu, DMAR_FEUADDR_REG0x44, iommu->feaddr >> 32);
2702
2703 mtx_leave(&iommu->reg_lock);
2704}
2705
2706void
2707acpidmar_msi_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec,
2708 int type)
2709{
2710 printf("msi_delroute\n");
2711}
2712
2713void *
2714acpidmar_intr_establish(void *ctx, int level, int (*func)(void *),
2715 void *arg, const char *what)
2716{
2717 struct iommu_softc *iommu = ctx;
2718 struct pic *pic;
2719
2720 pic = &iommu->pic.pic;
2721 iommu->pic.iommu = iommu;
2722
2723 strlcpy(pic->pic_dev.dv_xname, "dmarpic",
2724 sizeof(pic->pic_dev.dv_xname));
2725 pic->pic_type = PIC_MSI3;
2726 pic->pic_hwmask = acpidmar_msi_hwmask;
2727 pic->pic_hwunmask = acpidmar_msi_hwunmask;
2728 pic->pic_addroute = acpidmar_msi_addroute;
2729 pic->pic_delroute = acpidmar_msi_delroute;
2730 pic->pic_edge_stubs = ioapic_edge_stubs;
2731#ifdef MULTIPROCESSOR1
2732 mtx_init(&pic->pic_mutex, level)do { (void)(((void *)0)); (void)(0); __mtx_init((&pic->
pic_mutex), ((((level)) > 0x0 && ((level)) < 0x9
) ? 0x9 : ((level)))); } while (0);
2733#endif
2734
2735 return intr_establish(-1, pic, 0, IST_PULSE1, level, NULL((void *)0), func, arg, what);
2736}
2737
2738/* Intel: Handle DMAR Interrupt */
2739int
2740acpidmar_intr(void *ctx)
2741{
2742 struct iommu_softc *iommu = ctx;
2743 struct fault_entry fe;
2744 static struct fault_entry ofe;
2745 int fro, nfr, fri, i;
2746 uint32_t sts;
2747
2748 /*splassert(IPL_HIGH);*/
2749
2750 if (!(iommu->gcmd & GCMD_TE(1LL << 31))) {
2751 return (1);
2752 }
2753 mtx_enter(&iommu->reg_lock);
2754 sts = iommu_read_4(iommu, DMAR_FECTL_REG0x38);
2755 sts = iommu_read_4(iommu, DMAR_FSTS_REG0x34);
2756
2757 if (!(sts & FSTS_PPF(1LL << 1))) {
2758 mtx_leave(&iommu->reg_lock);
2759 return (1);
2760 }
2761
2762 nfr = cap_nfr(iommu->cap)((uint32_t)(((iommu->cap)>> 40LL) & 0xFF) + 1);
2763 fro = cap_fro(iommu->cap)((uint32_t)(((iommu->cap)>> 24LL) & 0x3FF) * 16);
2764 fri = (sts >> FSTS_FRI_SHIFT8) & FSTS_FRI_MASK0xFF;
2765 for (i = 0; i < nfr; i++) {
2766 fe.hi = iommu_read_8(iommu, fro + (fri*16) + 8);
2767 if (!(fe.hi & FRCD_HI_F(1LL << (127-64))))
2768 break;
2769
2770 fe.lo = iommu_read_8(iommu, fro + (fri*16));
2771 if (ofe.hi != fe.hi || ofe.lo != fe.lo) {
2772 iommu_showfault(iommu, fri, &fe);
2773 ofe.hi = fe.hi;
2774 ofe.lo = fe.lo;
2775 }
2776 fri = (fri + 1) % nfr;
2777 }
2778
2779 iommu_write_4(iommu, DMAR_FSTS_REG0x34, FSTS_PFO(1LL << 0) | FSTS_PPF(1LL << 1));
2780
2781 mtx_leave(&iommu->reg_lock);
2782
2783 return (1);
2784}
2785
2786const char *vtd_faults[] = {
2787 "Software",
2788 "Root Entry Not Present", /* ok (rtaddr + 4096) */
2789 "Context Entry Not Present", /* ok (no CTX_P) */
2790 "Context Entry Invalid", /* ok (tt = 3) */
2791 "Address Beyond MGAW",
2792 "Write", /* ok */
2793 "Read", /* ok */
2794 "Paging Entry Invalid", /* ok */
2795 "Root Table Invalid",
2796 "Context Table Invalid",
2797 "Root Entry Reserved", /* ok (root.lo |= 0x4) */
2798 "Context Entry Reserved",
2799 "Paging Entry Reserved",
2800 "Context Entry TT",
2801 "Reserved",
2802};
2803
2804void iommu_showpte(uint64_t, int, uint64_t);
2805
2806/* Intel: Show IOMMU page table entry */
2807void
2808iommu_showpte(uint64_t ptep, int lvl, uint64_t base)
2809{
2810 uint64_t nb, pb, i;
2811 struct pte_entry *pte;
2812
2813 pte = (void *)PMAP_DIRECT_MAP(ptep)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (ptep));
2814 for (i = 0; i < 512; i++) {
2815 if (!(pte[i].val & PTE_P(1L << 0)))
2816 continue;
2817 nb = base + (i << lvl);
2818 pb = pte[i].val & ~VTD_PAGE_MASK0xFFF;
2819 if(lvl == VTD_LEVEL012) {
2820 printf(" %3llx %.16llx = %.16llx %c%c %s\n",
2821 i, nb, pb,
2822 pte[i].val == PTE_R0x00 ? 'r' : ' ',
2823 pte[i].val & PTE_W(1L << 1) ? 'w' : ' ',
2824 (nb == pb) ? " ident" : "");
2825 if (nb == pb)
2826 return;
2827 } else {
2828 iommu_showpte(pb, lvl - VTD_STRIDE_SIZE9, nb);
2829 }
2830 }
2831}
2832
2833/* Intel: Show IOMMU configuration */
2834void
2835iommu_showcfg(struct iommu_softc *iommu, int sid)
2836{
2837 int i, j, sts, cmd;
2838 struct context_entry *ctx;
2839 pcitag_t tag;
2840 pcireg_t clc;
2841
2842 cmd = iommu_read_4(iommu, DMAR_GCMD_REG0x18);
2843 sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
2844 printf("iommu%d: flags:%d root pa:%.16llx %s %s %s %.8x %.8x\n",
2845 iommu->id, iommu->flags, iommu_read_8(iommu, DMAR_RTADDR_REG0x20),
2846 sts & GSTS_TES(1LL << 31) ? "enabled" : "disabled",
2847 sts & GSTS_QIES(1LL << 26) ? "qi" : "ccmd",
2848 sts & GSTS_IRES(1LL << 25) ? "ir" : "",
2849 cmd, sts);
2850 for (i = 0; i < 256; i++) {
2851 if (!root_entry_is_valid(&iommu->root[i])) {
2852 continue;
2853 }
2854 for (j = 0; j < 256; j++) {
2855 ctx = iommu->ctx[i] + j;
2856 if (!context_entry_is_valid(ctx)) {
2857 continue;
2858 }
2859 tag = pci_make_tag(NULL((void *)0), i, (j >> 3), j & 0x7);
2860 clc = pci_conf_read(NULL((void *)0), tag, 0x08) >> 8;
2861 printf(" %.2x:%.2x.%x lvl:%d did:%.4x tt:%d ptep:%.16llx flag:%x cc:%.6x\n",
2862 i, (j >> 3), j & 7,
2863 context_address_width(ctx),
2864 context_domain_id(ctx),
2865 context_translation_type(ctx),
2866 context_pte(ctx),
2867 context_user(ctx),
2868 clc);
2869#if 0
2870 /* dump pagetables */
2871 iommu_showpte(ctx->lo & ~VTD_PAGE_MASK0xFFF, iommu->agaw -
2872 VTD_STRIDE_SIZE9, 0);
2873#endif
2874 }
2875 }
2876}
2877
2878/* Intel: Show IOMMU fault */
2879void
2880iommu_showfault(struct iommu_softc *iommu, int fri, struct fault_entry *fe)
2881{
2882 int bus, dev, fun, type, fr, df;
2883 bios_memmap_t *im;
2884 const char *mapped;
2885
2886 if (!(fe->hi & FRCD_HI_F(1LL << (127-64))))
2887 return;
2888 type = (fe->hi & FRCD_HI_T(1LL << (126-64))) ? 'r' : 'w';
2889 fr = (fe->hi >> FRCD_HI_FR_SHIFT(96-64)) & FRCD_HI_FR_MASK0xFF;
2890 bus = (fe->hi >> FRCD_HI_BUS_SHIFT8) & FRCD_HI_BUS_MASK0xFF;
2891 dev = (fe->hi >> FRCD_HI_DEV_SHIFT3) & FRCD_HI_DEV_MASK0x1F;
2892 fun = (fe->hi >> FRCD_HI_FUN_SHIFT0) & FRCD_HI_FUN_MASK0x7;
2893 df = (fe->hi >> FRCD_HI_FUN_SHIFT0) & 0xFF;
2894 iommu_showcfg(iommu, mksid(bus,dev,fun));
2895 if (!iommu->ctx[bus]) {
2896 /* Bus is not initialized */
2897 mapped = "nobus";
2898 } else if (!context_entry_is_valid(&iommu->ctx[bus][df])) {
2899 /* DevFn not initialized */
2900 mapped = "nodevfn";
2901 } else if (context_user(&iommu->ctx[bus][df]) != 0xA) {
2902 /* no bus_space_map */
2903 mapped = "nomap";
2904 } else {
2905 /* bus_space_map */
2906 mapped = "mapped";
2907 }
2908 printf("fri%d: dmar: %.2x:%.2x.%x %s error at %llx fr:%d [%s] iommu:%d [%s]\n",
2909 fri, bus, dev, fun,
2910 type == 'r' ? "read" : "write",
2911 fe->lo,
2912 fr, fr <= 13 ? vtd_faults[fr] : "unknown",
2913 iommu->id,
2914 mapped);
2915 for (im = bios_memmap; im->type != BIOS_MAP_END0x00; im++) {
2916 if ((im->type == BIOS_MAP_RES0x02) &&
2917 (im->addr <= fe->lo) &&
2918 (fe->lo <= im->addr+im->size)) {
2919 printf("mem in e820.reserved\n");
2920 }
2921 }
2922#ifdef DDB1
2923 if (acpidmar_ddb)
2924 db_enter();
2925#endif
2926}
2927