/usr/src/sys/dev/acpi/acpidmar.c

Bug Summary

File:	dev/acpi/acpidmar.c
Warning:	line 1898, column 5 Value stored to 'rc' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name acpidmar.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/acpi/acpidmar.c

1	/*
2	* Copyright (c) 2015 Jordan Hargrave <jordan_hargrave@hotmail.com>
3	*
4	* Permission to use, copy, modify, and distribute this software for any
5	* purpose with or without fee is hereby granted, provided that the above
6	* copyright notice and this permission notice appear in all copies.
7	*
8	* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9	* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10	* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11	* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13	* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14	* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15	*/
16
17	#include <sys/param.h>
18	#include <sys/systm.h>
19	#include <sys/kernel.h>
20	#include <sys/device.h>
21	#include <sys/malloc.h>
22	#include <sys/queue.h>
23	#include <sys/types.h>
24	#include <sys/mbuf.h>
25	#include <sys/proc.h>
26
27	#include <uvm/uvm_extern.h>
28
29	#include <machine/apicvar.h>
30	#include <machine/biosvar.h>
31	#include <machine/cpuvar.h>
32	#include <machine/bus.h>
33
34	#include <dev/acpi/acpireg.h>
35	#include <dev/acpi/acpivar.h>
36	#include <dev/acpi/acpidev.h>
37	#include <dev/acpi/amltypes.h>
38	#include <dev/acpi/dsdt.h>
39
40	#include <machine/i8259.h>
41	#include <machine/i82093reg.h>
42	#include <machine/i82093var.h>
43	#include <machine/i82489reg.h>
44	#include <machine/i82489var.h>
45
46	#include <machine/mpbiosvar.h>
47
48	#include <dev/pci/pcireg.h>
49	#include <dev/pci/pcivar.h>
50	#include <dev/pci/pcidevs.h>
51	#include <dev/pci/ppbreg.h>
52
53	#include "ioapic.h"
54
55	#include "acpidmar.h"
56	#include "amd_iommu.h"
57
58	/* We don't want IOMMU to remap MSI */
59	#define MSI_BASE_ADDRESS0xFEE00000L 0xFEE00000L
60	#define MSI_BASE_SIZE0x00100000L 0x00100000L
61	#define MAX_DEVFN65536 65536
62
63	#ifdef IOMMU_DEBUG
64	int acpidmar_dbg_lvl = 0;
65	#define DPRINTF(lvl,x...) if (acpidmar_dbg_lvl >= lvl) { printf(x); }
66	#else
67	#define DPRINTF(lvl,x...)
68	#endif
69
70	#ifdef DDB1
71	int acpidmar_ddb = 0;
72	#endif
73
74	int acpidmar_force_cm = 1;
75
76	/* Page Table Entry per domain */
77	struct iommu_softc;
78
79	static inline int
80	mksid(int b, int d, int f)
81	{
82	return (b << 8) + (d << 3) + f;
83	}
84
85	static inline int
86	sid_devfn(int sid)
87	{
88	return sid & 0xff;
89	}
90
91	static inline int
92	sid_bus(int sid)
93	{
94	return (sid >> 8) & 0xff;
95	}
96
97	static inline int
98	sid_dev(int sid)
99	{
100	return (sid >> 3) & 0x1f;
101	}
102
103	static inline int
104	sid_fun(int sid)
105	{
106	return (sid >> 0) & 0x7;
107	}
108
109	/* Alias mapping */
110	#define SID_INVALID0x80000000L 0x80000000L
111	static uint32_t sid_flag[MAX_DEVFN65536];
112
113	struct domain_dev {
114	int sid;
115	int sec;
116	int sub;
117	TAILQ_ENTRY(domain_dev)struct { struct domain_dev tqe_next; struct domain_dev *tqe_prev ; } link;
118	};
119
120	struct domain {
121	struct iommu_softc *iommu;
122	int did;
123	int gaw;
124	struct pte_entry *pte;
125	paddr_t ptep;
126	struct bus_dma_tag dmat;
127	int flag;
128
129	struct mutex exlck;
130	char exname[32];
131	struct extent *iovamap;
132	TAILQ_HEAD(,domain_dev)struct { struct domain_dev tqh_first; struct domain_dev *tqh_last ; } devices;
133	TAILQ_ENTRY(domain)struct { struct domain tqe_next; struct domain *tqe_prev; } link;
134	};
135
136	#define DOM_DEBUG0x1 0x1
137	#define DOM_NOMAP0x2 0x2
138
139	struct dmar_devlist {
140	int type;
141	int bus;
142	int ndp;
143	struct acpidmar_devpath *dp;
144	TAILQ_ENTRY(dmar_devlist)struct { struct dmar_devlist tqe_next; struct dmar_devlist *tqe_prev; } link;
145	};
146
147	TAILQ_HEAD(devlist_head, dmar_devlist)struct devlist_head { struct dmar_devlist tqh_first; struct dmar_devlist *tqh_last; };
148
149	struct ivhd_devlist {
150	int start_id;
151	int end_id;
152	int cfg;
153	TAILQ_ENTRY(ivhd_devlist)struct { struct ivhd_devlist tqe_next; struct ivhd_devlist *tqe_prev; } link;
154	};
155
156	struct rmrr_softc {
157	TAILQ_ENTRY(rmrr_softc)struct { struct rmrr_softc tqe_next; struct rmrr_softc *tqe_prev ; } link;
158	struct devlist_head devices;
159	int segment;
160	uint64_t start;
161	uint64_t end;
162	};
163
164	struct atsr_softc {
165	TAILQ_ENTRY(atsr_softc)struct { struct atsr_softc tqe_next; struct atsr_softc *tqe_prev ; } link;
166	struct devlist_head devices;
167	int segment;
168	int flags;
169	};
170
171	struct iommu_pic {
172	struct pic pic;
173	struct iommu_softc *iommu;
174	};
175
176	#define IOMMU_FLAGS_CATCHALL0x1 0x1
177	#define IOMMU_FLAGS_BAD0x2 0x2
178	#define IOMMU_FLAGS_SUSPEND0x4 0x4
179
180	struct iommu_softc {
181	TAILQ_ENTRY(iommu_softc)struct { struct iommu_softc tqe_next; struct iommu_softc *tqe_prev ; }link;
182	struct devlist_head devices;
183	int id;
184	int flags;
185	int segment;
186
187	struct mutex reg_lock;
188
189	bus_space_tag_t iot;
190	bus_space_handle_t ioh;
191
192	uint64_t cap;
193	uint64_t ecap;
194	uint32_t gcmd;
195
196	int mgaw;
197	int agaw;
198	int ndoms;
199
200	struct root_entry *root;
201	struct context_entry *ctx[256];
202
203	void *intr;
204	struct iommu_pic pic;
205	int fedata;
206	uint64_t feaddr;
207	uint64_t rtaddr;
208
209	/* Queued Invalidation */
210	int qi_head;
211	int qi_tail;
212	paddr_t qip;
213	struct qi_entry *qi;
214
215	struct domain *unity;
216	TAILQ_HEAD(,domain)struct { struct domain tqh_first; struct domain *tqh_last; } domains;
217
218	/* AMD iommu */
219	struct ivhd_dte *dte;
220	void *cmd_tbl;
221	void *evt_tbl;
222	paddr_t cmd_tblp;
223	paddr_t evt_tblp;
224	};
225
226	static inline int iommu_bad(struct iommu_softc *sc)
227	{
228	return (sc->flags & IOMMU_FLAGS_BAD0x2);
229	}
230
231	static inline int iommu_enabled(struct iommu_softc *sc)
232	{
233	if (sc->dte) {
234	return 1;
235	}
236	return (sc->gcmd & GCMD_TE(1LL << 31));
237	}
238
239	struct acpidmar_softc {
240	struct device sc_dev;
241
242	pci_chipset_tag_t sc_pc;
243	bus_space_tag_t sc_memt;
244	int sc_haw;
245	int sc_flags;
246	bus_dma_tag_t sc_dmat;
247
248	struct ivhd_dte *sc_hwdte;
249	paddr_t sc_hwdtep;
250
251	TAILQ_HEAD(,iommu_softc)struct { struct iommu_softc tqh_first; struct iommu_softc * tqh_last; }sc_drhds;
252	TAILQ_HEAD(,rmrr_softc)struct { struct rmrr_softc tqh_first; struct rmrr_softc *tqh_last ; } sc_rmrrs;
253	TAILQ_HEAD(,atsr_softc)struct { struct atsr_softc tqh_first; struct atsr_softc *tqh_last ; } sc_atsrs;
254	};
255
256	int acpidmar_activate(struct device *, int);
257	int acpidmar_match(struct device , void , void *);
258	void acpidmar_attach(struct device , struct device , void *);
259	struct domain acpidmar_pci_attach(struct acpidmar_softc , int, int, int);
260
261	const struct cfattach acpidmar_ca = {
262	sizeof(struct acpidmar_softc), acpidmar_match, acpidmar_attach, NULL((void *)0),
263	acpidmar_activate
264	};
265
266	struct cfdriver acpidmar_cd = {
267	NULL((void *)0), "acpidmar", DV_DULL
268	};
269
270	struct acpidmar_softc *acpidmar_sc;
271	int acpidmar_intr(void *);
272	int acpiivhd_intr(void *);
273
274	#define DID_UNITY0x1 0x1
275
276	void _dumppte(struct pte_entry *, int, vaddr_t);
277
278	struct domain domain_create(struct iommu_softc , int);
279	struct domain domain_lookup(struct acpidmar_softc , int, int);
280
281	void domain_unload_map(struct domain *, bus_dmamap_t);
282	void domain_load_map(struct domain , bus_dmamap_t, int, int, const char );
283
284	void (domain_map_page)(struct domain , vaddr_t, paddr_t, uint64_t);
285	void domain_map_page_amd(struct domain *, vaddr_t, paddr_t, uint64_t);
286	void domain_map_page_intel(struct domain *, vaddr_t, paddr_t, uint64_t);
287	void domain_map_pthru(struct domain *, paddr_t, paddr_t);
288
289	void acpidmar_pci_hook(pci_chipset_tag_t, struct pci_attach_args *);
290	void acpidmar_parse_devscope(union acpidmar_entry *, int, int,
291	struct devlist_head *);
292	int acpidmar_match_devscope(struct devlist_head *, pci_chipset_tag_t, int);
293
294	void acpidmar_init(struct acpidmar_softc , struct acpi_dmar );
295	void acpidmar_drhd(struct acpidmar_softc , union acpidmar_entry );
296	void acpidmar_rmrr(struct acpidmar_softc , union acpidmar_entry );
297	void acpidmar_atsr(struct acpidmar_softc , union acpidmar_entry );
298	void acpiivrs_init(struct acpidmar_softc , struct acpi_ivrs );
299
300	void acpidmar_intr_establish(void , int, int ()(void ), void *,
301	const char *);
302
303	void iommu_write_4(struct iommu_softc *, int, uint32_t);
304	uint32_t iommu_read_4(struct iommu_softc *, int);
305	void iommu_write_8(struct iommu_softc *, int, uint64_t);
306	uint64_t iommu_read_8(struct iommu_softc *, int);
307	void iommu_showfault(struct iommu_softc *, int,
308	struct fault_entry *);
309	void iommu_showcfg(struct iommu_softc *, int);
310
311	int iommu_init(struct acpidmar_softc , struct iommu_softc ,
312	struct acpidmar_drhd *);
313	int iommu_enable_translation(struct iommu_softc *, int);
314	void iommu_enable_qi(struct iommu_softc *, int);
315	void iommu_flush_cache(struct iommu_softc , void , size_t);
316	void iommu_alloc_page(struct iommu_softc , paddr_t *);
317	void iommu_flush_write_buffer(struct iommu_softc *);
318	void iommu_issue_qi(struct iommu_softc , struct qi_entry );
319
320	void iommu_flush_ctx(struct iommu_softc *, int, int, int, int);
321	void iommu_flush_ctx_qi(struct iommu_softc *, int, int, int, int);
322	void iommu_flush_tlb(struct iommu_softc *, int, int);
323	void iommu_flush_tlb_qi(struct iommu_softc *, int, int);
324
325	void iommu_set_rtaddr(struct iommu_softc *, paddr_t);
326
327	void iommu_alloc_hwdte(struct acpidmar_softc , size_t, paddr_t *);
328
329	const char *dmar_bdf(int);
330
331	const char *
332	dmar_bdf(int sid)
333	{
334	static char bdf[32];
335
336	snprintf(bdf, sizeof(bdf), "%.4x:%.2x:%.2x.%x", 0,
337	sid_bus(sid), sid_dev(sid), sid_fun(sid));
338
339	return (bdf);
340	}
341
342	/* busdma */
343	static int dmar_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
344	bus_size_t, int, bus_dmamap_t *);
345	static void dmar_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
346	static int dmar_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
347	struct proc *, int);
348	static int dmar_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *,
349	int);
350	static int dmar_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t, struct uio *, int);
351	static int dmar_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t,
352	bus_dma_segment_t *, int, bus_size_t, int);
353	static void dmar_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
354	static void dmar_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
355	bus_size_t, int);
356	static int dmar_dmamem_alloc(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
357	bus_dma_segment_t , int, int , int);
358	static void dmar_dmamem_free(bus_dma_tag_t, bus_dma_segment_t *, int);
359	static int dmar_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *, int, size_t,
360	caddr_t *, int);
361	static void dmar_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
362	static paddr_t dmar_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, int, off_t,
363	int, int);
364
365	static void dmar_dumpseg(bus_dma_tag_t, int, bus_dma_segment_t , const char );
366	const char dom_bdf(struct domain );
367	void domain_map_check(struct domain *);
368
369	struct pte_entry pte_lvl(struct iommu_softc , struct pte_entry *, vaddr_t, int, uint64_t);
370	int ivhd_poll_events(struct iommu_softc *);
371	void ivhd_showreg(struct iommu_softc *);
372	void ivhd_showdte(struct iommu_softc *);
373	void ivhd_showcmd(struct iommu_softc *);
374
375	static inline int
376	debugme(struct domain *dom)
377	{
378	return 0;
379	return (dom->flag & DOM_DEBUG0x1);
380	}
381
382	void
383	domain_map_check(struct domain *dom)
384	{
385	struct iommu_softc *iommu;
386	struct domain_dev *dd;
387	struct context_entry *ctx;
388	int v;
389
390	iommu = dom->iommu;
391	TAILQ_FOREACH(dd, &dom->devices, link)for((dd) = ((&dom->devices)->tqh_first); (dd) != (( void *)0); (dd) = ((dd)->link.tqe_next)) {
392	acpidmar_pci_attach(acpidmar_sc, iommu->segment, dd->sid, 1);
393
394	if (iommu->dte)
395	continue;
396
397	/* Check if this is the first time we are mapped */
398	ctx = &iommu->ctx[sid_bus(dd->sid)][sid_devfn(dd->sid)];
399	v = context_user(ctx);
400	if (v != 0xA) {
401	printf(" map: %.4x:%.2x:%.2x.%x iommu:%d did:%.4x\n",
402	iommu->segment,
403	sid_bus(dd->sid),
404	sid_dev(dd->sid),
405	sid_fun(dd->sid),
406	iommu->id,
407	dom->did);
408	context_set_user(ctx, 0xA);
409	}
410	}
411	}
412
413	/* Map a single page as passthrough - used for DRM */
414	void
415	dmar_ptmap(bus_dma_tag_t tag, bus_addr_t addr)
416	{
417	struct domain *dom = tag->_cookie;
418
419	if (!acpidmar_sc)
420	return;
421	domain_map_check(dom);
422	domain_map_page(dom, addr, addr, PTE_P(1L << 0) \| PTE_R0x00 \| PTE_W(1L << 1));
423	}
424
425	/* Map a range of pages 1:1 */
426	void
427	domain_map_pthru(struct domain *dom, paddr_t start, paddr_t end)
428	{
429	domain_map_check(dom);
430	while (start < end) {
431	domain_map_page(dom, start, start, PTE_P(1L << 0) \| PTE_R0x00 \| PTE_W(1L << 1));
432	start += VTD_PAGE_SIZE4096;
433	}
434	}
435
436	/* Map a single paddr to IOMMU paddr */
437	void
438	domain_map_page_intel(struct domain *dom, vaddr_t va, paddr_t pa, uint64_t flags)
439	{
440	paddr_t paddr;
441	struct pte_entry pte, npte;
442	int lvl, idx;
443	struct iommu_softc *iommu;
444
445	iommu = dom->iommu;
446	/* Insert physical address into virtual address map
447	* XXX: could we use private pmap here?
448	* essentially doing a pmap_enter(map, va, pa, prot);
449	*/
450
451	/* Only handle 4k pages for now */
452	npte = dom->pte;
453	for (lvl = iommu->agaw - VTD_STRIDE_SIZE9; lvl>= VTD_LEVEL012;
454	lvl -= VTD_STRIDE_SIZE9) {
455	idx = (va >> lvl) & VTD_STRIDE_MASK0x1FF;
456	pte = &npte[idx];
457	if (lvl == VTD_LEVEL012) {
458	/* Level 1: Page Table - add physical address */
459	pte->val = pa \| flags;
460	iommu_flush_cache(iommu, pte, sizeof(*pte));
461	break;
462	} else if (!(pte->val & PTE_P(1L << 0))) {
463	/* Level N: Point to lower level table */
464	iommu_alloc_page(iommu, &paddr);
465	pte->val = paddr \| PTE_P(1L << 0) \| PTE_R0x00 \| PTE_W(1L << 1);
466	iommu_flush_cache(iommu, pte, sizeof(*pte));
467	}
468	npte = (void )PMAP_DIRECT_MAP((pte->val & VTD_PTE_MASK))((vaddr_t)(((((511 - 4) (1ULL << 39))) \| 0xffff000000000000 )) + ((pte->val & 0x0000FFFFFFFFF000LL)));
469	}
470	}
471
472	/* Map a single paddr to IOMMU paddr: AMD
473	* physical address breakdown into levels:
474	* xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
475	* 5.55555555.44444444.43333333,33222222.22211111.1111----.--------
476	* mode:
477	* 000 = none shift
478	* 001 = 1 [21].12
479	* 010 = 2 [30].21
480	* 011 = 3 [39].30
481	* 100 = 4 [48].39
482	* 101 = 5 [57]
483	* 110 = 6
484	* 111 = reserved
485	*/
486	struct pte_entry *
487	pte_lvl(struct iommu_softc iommu, struct pte_entry pte, vaddr_t va,
488	int shift, uint64_t flags)
489	{
490	paddr_t paddr;
491	int idx;
492
493	idx = (va >> shift) & VTD_STRIDE_MASK0x1FF;
494	if (!(pte[idx].val & PTE_P(1L << 0))) {
495	/* Page Table entry is not present... create a new page entry */
496	iommu_alloc_page(iommu, &paddr);
497	pte[idx].val = paddr \| flags;
498	iommu_flush_cache(iommu, &pte[idx], sizeof(pte[idx]));
499	}
500	return (void )PMAP_DIRECT_MAP((pte[idx].val & PTE_PADDR_MASK))((vaddr_t)(((((511 - 4) (1ULL << 39))) \| 0xffff000000000000 )) + ((pte[idx].val & 0x000FFFFFFFFFF000LL)));
501	}
502
503	void
504	domain_map_page_amd(struct domain *dom, vaddr_t va, paddr_t pa, uint64_t flags)
505	{
506	struct pte_entry *pte;
507	struct iommu_softc *iommu;
508	int idx;
509
510	iommu = dom->iommu;
511	/* Insert physical address into virtual address map
512	* XXX: could we use private pmap here?
513	* essentially doing a pmap_enter(map, va, pa, prot);
514	*/
515
516	/* Always assume AMD levels=4 */
517	/* 39 30 21 12 */
518	/* ---------\|---------\|---------\|---------\|------------ */
519	pte = dom->pte;
520	pte = pte_lvl(iommu, pte, va, 30, PTE_NXTLVL(2)(((2) & 0x7) << 9) \| PTE_IR(1LL << 61) \| PTE_IW(1LL << 62) \| PTE_P(1L << 0));
521	pte = pte_lvl(iommu, pte, va, 21, PTE_NXTLVL(1)(((1) & 0x7) << 9) \| PTE_IR(1LL << 61) \| PTE_IW(1LL << 62) \| PTE_P(1L << 0));
522	if (flags)
523	flags = PTE_P(1L << 0) \| PTE_R0x00 \| PTE_W(1L << 1) \| PTE_IW(1LL << 62) \| PTE_IR(1LL << 61) \| PTE_NXTLVL(0)(((0) & 0x7) << 9);
524
525	/* Level 1: Page Table - add physical address */
526	idx = (va >> 12) & 0x1FF;
527	pte[idx].val = pa \| flags;
528
529	iommu_flush_cache(iommu, pte, sizeof(*pte));
530	}
531
532	static void
533	dmar_dumpseg(bus_dma_tag_t tag, int nseg, bus_dma_segment_t *segs,
534	const char *lbl)
535	{
536	struct domain *dom = tag->_cookie;
537	int i;
538
539	return;
540	if (!debugme(dom))
541	return;
542	printf("%s: %s\n", lbl, dom_bdf(dom));
543	for (i = 0; i < nseg; i++) {
544	printf(" %.16llx %.8x\n",
545	(uint64_t)segs[i].ds_addr,
546	(uint32_t)segs[i].ds_len);
547	}
548	}
549
550	/* Unload mapping */
551	void
552	domain_unload_map(struct domain *dom, bus_dmamap_t dmam)
553	{
554	bus_dma_segment_t *seg;
555	paddr_t base, end, idx;
556	psize_t alen;
557	int i;
558
559	if (iommu_bad(dom->iommu)) {
560	printf("unload map no iommu\n");
561	return;
562	}
563
564	for (i = 0; i < dmam->dm_nsegs; i++) {
565	seg = &dmam->dm_segs[i];
566
567	base = trunc_page(seg->ds_addr)((seg->ds_addr) & ~((1 << 12) - 1));
568	end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE)((((seg->ds_addr + seg->ds_len)+((4096)-1))/(4096))*(4096 ));
569	alen = end - base;
570
571	if (debugme(dom)) {
572	printf(" va:%.16llx len:%x\n",
573	(uint64_t)base, (uint32_t)alen);
574	}
575
576	/* Clear PTE */
577	for (idx = 0; idx < alen; idx += VTD_PAGE_SIZE4096)
578	domain_map_page(dom, base + idx, 0, 0);
579
580	if (dom->flag & DOM_NOMAP0x2) {
581	printf("%s: nomap %.16llx\n", dom_bdf(dom), (uint64_t)base);
582	continue;
583	}
584
585	mtx_enter(&dom->exlck);
586	if (extent_free(dom->iovamap, base, alen, EX_NOWAIT0x0000)) {
587	panic("domain_unload_map: extent_free");
588	}
589	mtx_leave(&dom->exlck);
590	}
591	}
592
593	/* map.segs[x].ds_addr is modified to IOMMU virtual PA */
594	void
595	domain_load_map(struct domain dom, bus_dmamap_t map, int flags, int pteflag, const char fn)
596	{
597	bus_dma_segment_t *seg;
598	struct iommu_softc *iommu;
599	paddr_t base, end, idx;
600	psize_t alen;
601	u_long res;
602	int i;
603
604	iommu = dom->iommu;
605	if (!iommu_enabled(iommu)) {
606	/* Lazy enable translation when required */
607	if (iommu_enable_translation(iommu, 1)) {
608	return;
609	}
610	}
611	domain_map_check(dom);
612	for (i = 0; i < map->dm_nsegs; i++) {
613	seg = &map->dm_segs[i];
614
615	base = trunc_page(seg->ds_addr)((seg->ds_addr) & ~((1 << 12) - 1));
616	end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE)((((seg->ds_addr + seg->ds_len)+((4096)-1))/(4096))*(4096 ));
617	alen = end - base;
618	res = base;
619
620	if (dom->flag & DOM_NOMAP0x2) {
621	goto nomap;
622	}
623
624	/* Allocate DMA Virtual Address */
625	mtx_enter(&dom->exlck);
626	if (extent_alloc(dom->iovamap, alen, VTD_PAGE_SIZE, 0,extent_alloc_subregion((dom->iovamap), (dom->iovamap)-> ex_start, (dom->iovamap)->ex_end, (alen), (4096), (0), ( map->_dm_boundary), (0x0000), (&res))
627	map->_dm_boundary, EX_NOWAIT, &res)extent_alloc_subregion((dom->iovamap), (dom->iovamap)-> ex_start, (dom->iovamap)->ex_end, (alen), (4096), (0), ( map->_dm_boundary), (0x0000), (&res))) {
628	panic("domain_load_map: extent_alloc");
629	}
630	if (res == -1) {
631	panic("got -1 address");
632	}
633	mtx_leave(&dom->exlck);
634
635	/* Reassign DMA address */
636	seg->ds_addr = res \| (seg->ds_addr & VTD_PAGE_MASK0xFFF);
637	nomap:
638	if (debugme(dom)) {
639	printf(" LOADMAP: %.16llx %x => %.16llx\n",
640	(uint64_t)seg->ds_addr, (uint32_t)seg->ds_len,
641	(uint64_t)res);
642	}
643	for (idx = 0; idx < alen; idx += VTD_PAGE_SIZE4096) {
644	domain_map_page(dom, res + idx, base + idx,
645	PTE_P(1L << 0) \| pteflag);
646	}
647	}
648	if ((iommu->cap & CAP_CM(1LL << 7)) \|\| acpidmar_force_cm) {
649	iommu_flush_tlb(iommu, IOTLB_DOMAIN, dom->did);
650	} else {
651	iommu_flush_write_buffer(iommu);
652	}
653	}
654
655	const char *
656	dom_bdf(struct domain *dom)
657	{
658	struct domain_dev *dd;
659	static char mmm[48];
660
661	dd = TAILQ_FIRST(&dom->devices)((&dom->devices)->tqh_first);
662	snprintf(mmm, sizeof(mmm), "%s iommu:%d did:%.4x%s",
663	dmar_bdf(dd->sid), dom->iommu->id, dom->did,
664	dom->did == DID_UNITY0x1 ? " [unity]" : "");
665	return (mmm);
666	}
667
668	/* Bus DMA Map functions */
669	static int
670	dmar_dmamap_create(bus_dma_tag_t tag, bus_size_t size, int nsegments,
671	bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
672	{
673	int rc;
674
675	rc = _bus_dmamap_create(tag, size, nsegments, maxsegsz, boundary,
676	flags, dmamp);
677	if (!rc) {
678	dmar_dumpseg(tag, (dmamp)->dm_nsegs, (dmamp)->dm_segs,
679	__FUNCTION__);
680	}
681	return (rc);
682	}
683
684	static void
685	dmar_dmamap_destroy(bus_dma_tag_t tag, bus_dmamap_t dmam)
686	{
687	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
688	_bus_dmamap_destroy(tag, dmam);
689	}
690
691	static int
692	dmar_dmamap_load(bus_dma_tag_t tag, bus_dmamap_t dmam, void *buf,
693	bus_size_t buflen, struct proc *p, int flags)
694	{
695	struct domain *dom = tag->_cookie;
696	int rc;
697
698	rc = _bus_dmamap_load(tag, dmam, buf, buflen, p, flags);
699	if (!rc) {
700	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
701	__FUNCTION__);
702	domain_load_map(dom, dmam, flags, PTE_R0x00\|PTE_W(1L << 1), __FUNCTION__);
703	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
704	__FUNCTION__);
705	}
706	return (rc);
707	}
708
709	static int
710	dmar_dmamap_load_mbuf(bus_dma_tag_t tag, bus_dmamap_t dmam, struct mbuf *chain,
711	int flags)
712	{
713	struct domain *dom = tag->_cookie;
714	int rc;
715
716	rc = _bus_dmamap_load_mbuf(tag, dmam, chain, flags);
717	if (!rc) {
718	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
719	__FUNCTION__);
720	domain_load_map(dom, dmam, flags, PTE_R0x00\|PTE_W(1L << 1),__FUNCTION__);
721	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
722	__FUNCTION__);
723	}
724	return (rc);
725	}
726
727	static int
728	dmar_dmamap_load_uio(bus_dma_tag_t tag, bus_dmamap_t dmam, struct uio *uio,
729	int flags)
730	{
731	struct domain *dom = tag->_cookie;
732	int rc;
733
734	rc = _bus_dmamap_load_uio(tag, dmam, uio, flags);
735	if (!rc) {
736	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
737	__FUNCTION__);
738	domain_load_map(dom, dmam, flags, PTE_R0x00\|PTE_W(1L << 1), __FUNCTION__);
739	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
740	__FUNCTION__);
741	}
742	return (rc);
743	}
744
745	static int
746	dmar_dmamap_load_raw(bus_dma_tag_t tag, bus_dmamap_t dmam,
747	bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
748	{
749	struct domain *dom = tag->_cookie;
750	int rc;
751
752	rc = _bus_dmamap_load_raw(tag, dmam, segs, nsegs, size, flags);
753	if (!rc) {
754	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
755	__FUNCTION__);
756	domain_load_map(dom, dmam, flags, PTE_R0x00\|PTE_W(1L << 1), __FUNCTION__);
757	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs,
758	__FUNCTION__);
759	}
760	return (rc);
761	}
762
763	static void
764	dmar_dmamap_unload(bus_dma_tag_t tag, bus_dmamap_t dmam)
765	{
766	struct domain *dom = tag->_cookie;
767
768	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
769	domain_unload_map(dom, dmam);
770	_bus_dmamap_unload(tag, dmam);
771	}
772
773	static void
774	dmar_dmamap_sync(bus_dma_tag_t tag, bus_dmamap_t dmam, bus_addr_t offset,
775	bus_size_t len, int ops)
776	{
777	#if 0
778	struct domain *dom = tag->_cookie;
779	int flag;
780
781	flag = PTE_P(1L << 0);
782	if (ops == BUS_DMASYNC_PREREAD0x01) {
783	/* make readable */
784	flag \|= PTE_R0x00;
785	}
786	else if (ops == BUS_DMASYNC_PREWRITE0x04) {
787	/* make writeable */
788	flag \|= PTE_W(1L << 1);
789	}
790	dmar_dumpseg(tag, dmam->dm_nsegs, dmam->dm_segs, __FUNCTION__);
791	#endif
792	_bus_dmamap_sync(tag, dmam, offset, len, ops);
793	}
794
795	static int
796	dmar_dmamem_alloc(bus_dma_tag_t tag, bus_size_t size, bus_size_t alignment,
797	bus_size_t boundary, bus_dma_segment_t segs, int nsegs, int rsegs,
798	int flags)
799	{
800	int rc;
801
802	rc = _bus_dmamem_alloc(tag, size, alignment, boundary, segs, nsegs,
803	rsegs, flags);
804	if (!rc) {
805	dmar_dumpseg(tag, *rsegs, segs, __FUNCTION__);
806	}
807	return (rc);
808	}
809
810	static void
811	dmar_dmamem_free(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs)
812	{
813	dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
814	_bus_dmamem_free(tag, segs, nsegs);
815	}
816
817	static int
818	dmar_dmamem_map(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
819	size_t size, caddr_t *kvap, int flags)
820	{
821	dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
822	return (_bus_dmamem_map(tag, segs, nsegs, size, kvap, flags));
823	}
824
825	static void
826	dmar_dmamem_unmap(bus_dma_tag_t tag, caddr_t kva, size_t size)
827	{
828	struct domain *dom = tag->_cookie;
829
830	if (debugme(dom)) {
831	printf("dmamap_unmap: %s\n", dom_bdf(dom));
832	}
833	_bus_dmamem_unmap(tag, kva, size);
834	}
835
836	static paddr_t
837	dmar_dmamem_mmap(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
838	off_t off, int prot, int flags)
839	{
840	dmar_dumpseg(tag, nsegs, segs, __FUNCTION__);
841	return (_bus_dmamem_mmap(tag, segs, nsegs, off, prot, flags));
842	}
843
844	/*===================================
845	* IOMMU code
846	===================================/
847
848	/* Intel: Set Context Root Address */
849	void
850	iommu_set_rtaddr(struct iommu_softc *iommu, paddr_t paddr)
851	{
852	int i, sts;
853
854	mtx_enter(&iommu->reg_lock);
855	iommu_write_8(iommu, DMAR_RTADDR_REG0x20, paddr);
856	iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd \| GCMD_SRTP(1LL << 30));
857	for (i = 0; i < 5; i++) {
858	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
859	if (sts & GSTS_RTPS(1LL << 30))
860	break;
861	}
862	mtx_leave(&iommu->reg_lock);
863
864	if (i == 5) {
865	printf("set_rtaddr fails\n");
866	}
867	}
868
869	/* Allocate contiguous memory (1Mb) for the Device Table Entries */
870	void *
871	iommu_alloc_hwdte(struct acpidmar_softc sc, size_t size, paddr_t paddr)
872	{
873	caddr_t vaddr;
874	bus_dmamap_t map;
875	bus_dma_segment_t seg;
876	bus_dma_tag_t dmat = sc->sc_dmat;
877	int rc, nsegs;
878
879	rc = _bus_dmamap_create(dmat, size, 1, size, 0,
880	BUS_DMA_NOWAIT0x0001, &map);
881	if (rc != 0) {
882	printf("hwdte_create fails\n");
883	return NULL((void *)0);
884	}
885	rc = _bus_dmamem_alloc(dmat, size, 4, 0, &seg, 1,
886	&nsegs, BUS_DMA_NOWAIT0x0001 \| BUS_DMA_ZERO0x1000);
887	if (rc != 0) {
888	printf("hwdte alloc fails\n");
889	return NULL((void *)0);
890	}
891	rc = _bus_dmamem_map(dmat, &seg, 1, size, &vaddr,
892	BUS_DMA_NOWAIT0x0001 \| BUS_DMA_COHERENT0x0004);
893	if (rc != 0) {
894	printf("hwdte map fails\n");
895	return NULL((void *)0);
896	}
897	rc = _bus_dmamap_load_raw(dmat, map, &seg, 1, size, BUS_DMA_NOWAIT0x0001);
898	if (rc != 0) {
899	printf("hwdte load raw fails\n");
900	return NULL((void *)0);
901	}
902	*paddr = map->dm_segs[0].ds_addr;
903	return vaddr;
904	}
905
906	/* COMMON: Allocate a new memory page */
907	void *
908	iommu_alloc_page(struct iommu_softc iommu, paddr_t paddr)
909	{
910	void *va;
911
912	*paddr = 0;
913	va = km_alloc(VTD_PAGE_SIZE4096, &kv_page, &kp_zero, &kd_nowait);
914	if (va == NULL((void *)0)) {
915	panic("can't allocate page");
916	}
917	pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)va, paddr);
918	return (va);
919	}
920
921
922	/* Intel: Issue command via queued invalidation */
923	void
924	iommu_issue_qi(struct iommu_softc iommu, struct qi_entry qi)
925	{
926	#if 0
927	struct qi_entry pi, pw;
928
929	idx = iommu->qi_head;
930	pi = &iommu->qi[idx];
931	pw = &iommu->qi[(idx+1) % MAXQ];
932	iommu->qi_head = (idx+2) % MAXQ;
933
934	memcpy(pw, &qi, sizeof(qi))__builtin_memcpy((pw), (&qi), (sizeof(qi)));
935	issue command;
936	while (pw->xxx)
937	;
938	#endif
939	}
940
941	/* Intel: Flush TLB entries, Queued Invalidation mode */
942	void
943	iommu_flush_tlb_qi(struct iommu_softc *iommu, int mode, int did)
944	{
945	struct qi_entry qi;
946
947	/* Use queued invalidation */
948	qi.hi = 0;
949	switch (mode) {
950	case IOTLB_GLOBAL:
951	qi.lo = QI_IOTLB0x2 \| QI_IOTLB_IG_GLOBAL(1 << 4);
952	break;
953	case IOTLB_DOMAIN:
954	qi.lo = QI_IOTLB0x2 \| QI_IOTLB_IG_DOMAIN(2 << 4) \|
955	QI_IOTLB_DID(did)(((uint64_t)(did) << 16));
956	break;
957	case IOTLB_PAGE:
958	qi.lo = QI_IOTLB0x2 \| QI_IOTLB_IG_PAGE(3 << 4) \| QI_IOTLB_DID(did)(((uint64_t)(did) << 16));
959	qi.hi = 0;
960	break;
961	}
962	if (iommu->cap & CAP_DRD(1LL << 55))
963	qi.lo \|= QI_IOTLB_DR(1LL << 6);
964	if (iommu->cap & CAP_DWD(1LL << 54))
965	qi.lo \|= QI_IOTLB_DW(1LL << 5);
966	iommu_issue_qi(iommu, &qi);
967	}
968
969	/* Intel: Flush Context entries, Queued Invalidation mode */
970	void
971	iommu_flush_ctx_qi(struct iommu_softc *iommu, int mode, int did,
972	int sid, int fm)
973	{
974	struct qi_entry qi;
975
976	/* Use queued invalidation */
977	qi.hi = 0;
978	switch (mode) {
979	case CTX_GLOBAL:
980	qi.lo = QI_CTX0x1 \| QI_CTX_IG_GLOBAL(CTX_GLOBAL << 4);
981	break;
982	case CTX_DOMAIN:
983	qi.lo = QI_CTX0x1 \| QI_CTX_IG_DOMAIN(CTX_DOMAIN << 4) \| QI_CTX_DID(did)(((uint64_t)(did) << 16));
984	break;
985	case CTX_DEVICE:
986	qi.lo = QI_CTX0x1 \| QI_CTX_IG_DEVICE(CTX_DEVICE << 4) \| QI_CTX_DID(did)(((uint64_t)(did) << 16)) \|
987	QI_CTX_SID(sid)(((uint64_t)(sid) << 32)) \| QI_CTX_FM(fm)(((uint64_t)(fm) << 48));
988	break;
989	}
990	iommu_issue_qi(iommu, &qi);
991	}
992
993	/* Intel: Flush write buffers */
994	void
995	iommu_flush_write_buffer(struct iommu_softc *iommu)
996	{
997	int i, sts;
998
999	if (iommu->dte)
1000	return;
1001	if (!(iommu->cap & CAP_RWBF(1LL << 4)))
1002	return;
1003	DPRINTF(1,"writebuf\n");
1004	iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd \| GCMD_WBF(1LL << 27));
1005	for (i = 0; i < 5; i++) {
1006	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1007	if (sts & GSTS_WBFS(1LL << 27))
1008	break;
1009	delay(10000)(*delay_func)(10000);
1010	}
1011	if (i == 5) {
1012	printf("write buffer flush fails\n");
1013	}
1014	}
1015
1016	void
1017	iommu_flush_cache(struct iommu_softc iommu, void addr, size_t size)
1018	{
1019	if (iommu->dte) {
1020	pmap_flush_cache((vaddr_t)addr, size);
1021	return;
1022	}
1023	if (!(iommu->ecap & ECAP_C(1LL << 0)))
1024	pmap_flush_cache((vaddr_t)addr, size);
1025	}
1026
1027	/*
1028	* Intel: Flush IOMMU TLB Entries
1029	* Flushing can occur globally, per domain or per page
1030	*/
1031	void
1032	iommu_flush_tlb(struct iommu_softc *iommu, int mode, int did)
1033	{
1034	int n;
1035	uint64_t val;
1036
1037	/* Call AMD */
1038	if (iommu->dte) {
1039	ivhd_invalidate_domain(iommu, did);
1040	return;
1041	}
1042	val = IOTLB_IVT(1LL << 63);
1043	switch (mode) {
1044	case IOTLB_GLOBAL:
1045	val \|= IIG_GLOBAL((uint64_t)(IOTLB_GLOBAL) << 60);
1046	break;
1047	case IOTLB_DOMAIN:
1048	val \|= IIG_DOMAIN((uint64_t)(IOTLB_DOMAIN) << 60) \| IOTLB_DID(did)((uint64_t)(did) << 32);
1049	break;
1050	case IOTLB_PAGE:
1051	val \|= IIG_PAGE((uint64_t)(IOTLB_PAGE) << 60) \| IOTLB_DID(did)((uint64_t)(did) << 32);
1052	break;
1053	}
1054
1055	/* Check for Read/Write Drain */
1056	if (iommu->cap & CAP_DRD(1LL << 55))
1057	val \|= IOTLB_DR(1LL << 49);
1058	if (iommu->cap & CAP_DWD(1LL << 54))
1059	val \|= IOTLB_DW(1LL << 48);
1060
1061	mtx_enter(&iommu->reg_lock);
1062
1063	iommu_write_8(iommu, DMAR_IOTLB_REG(iommu)(((uint32_t)((((iommu)->ecap)>> 0x8) & 0x3FF) * 16 ) + 8), val);
1064	n = 0;
1065	do {
1066	val = iommu_read_8(iommu, DMAR_IOTLB_REG(iommu)(((uint32_t)((((iommu)->ecap)>> 0x8) & 0x3FF) * 16 ) + 8));
1067	} while (n++ < 5 && val & IOTLB_IVT(1LL << 63));
1068
1069	mtx_leave(&iommu->reg_lock);
1070	}
1071
1072	/* Intel: Flush IOMMU settings
1073	* Flushes can occur globally, per domain, or per device
1074	*/
1075	void
1076	iommu_flush_ctx(struct iommu_softc *iommu, int mode, int did, int sid, int fm)
1077	{
1078	uint64_t val;
1079	int n;
1080
1081	if (iommu->dte)
1082	return;
1083	val = CCMD_ICC(1LL << 63);
1084	switch (mode) {
1085	case CTX_GLOBAL:
1086	val \|= CIG_GLOBAL((uint64_t)(CTX_GLOBAL) << 61);
1087	break;
1088	case CTX_DOMAIN:
1089	val \|= CIG_DOMAIN((uint64_t)(CTX_DOMAIN) << 61) \| CCMD_DID(did)(((did) << 0));
1090	break;
1091	case CTX_DEVICE:
1092	val \|= CIG_DEVICE((uint64_t)(CTX_DEVICE) << 61) \| CCMD_DID(did)(((did) << 0)) \|
1093	CCMD_SID(sid)(((sid) << 8)) \| CCMD_FM(fm)(((uint64_t)(fm) << 32));
1094	break;
1095	}
1096
1097	mtx_enter(&iommu->reg_lock);
1098
1099	n = 0;
1100	iommu_write_8(iommu, DMAR_CCMD_REG0x28, val);
1101	do {
1102	val = iommu_read_8(iommu, DMAR_CCMD_REG0x28);
1103	} while (n++ < 5 && val & CCMD_ICC(1LL << 63));
1104
1105	mtx_leave(&iommu->reg_lock);
1106	}
1107
1108	/* Intel: Enable Queued Invalidation */
1109	void
1110	iommu_enable_qi(struct iommu_softc *iommu, int enable)
1111	{
1112	int n = 0;
1113	int sts;
1114
1115	if (!(iommu->ecap & ECAP_QI(1LL << 1)))
1116	return;
1117
1118	if (enable) {
1119	iommu->gcmd \|= GCMD_QIE(1LL << 26);
1120
1121	mtx_enter(&iommu->reg_lock);
1122
1123	iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1124	do {
1125	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1126	} while (n++ < 5 && !(sts & GSTS_QIES(1LL << 26)));
1127
1128	mtx_leave(&iommu->reg_lock);
1129
1130	DPRINTF(1,"set.qie: %d\n", n);
1131	} else {
1132	iommu->gcmd &= ~GCMD_QIE(1LL << 26);
1133
1134	mtx_enter(&iommu->reg_lock);
1135
1136	iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1137	do {
1138	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1139	} while (n++ < 5 && sts & GSTS_QIES(1LL << 26));
1140
1141	mtx_leave(&iommu->reg_lock);
1142
1143	DPRINTF(1,"clr.qie: %d\n", n);
1144	}
1145	}
1146
1147	/* Intel: Enable IOMMU translation */
1148	int
1149	iommu_enable_translation(struct iommu_softc *iommu, int enable)
1150	{
1151	uint32_t sts;
1152	uint64_t reg;
1153	int n = 0;
1154
1155	if (iommu->dte)
1156	return (0);
1157	reg = 0;
1158	if (enable) {
1159	DPRINTF(0,"enable iommu %d\n", iommu->id);
1160	iommu_showcfg(iommu, -1);
1161
1162	iommu->gcmd \|= GCMD_TE(1LL << 31);
1163
1164	/* Enable translation */
1165	printf(" pre tes: ");
1166
1167	mtx_enter(&iommu->reg_lock);
1168	iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1169	printf("xxx");
1170	do {
1171	printf("yyy");
1172	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1173	delay(n * 10000)(delay_func)(n 10000);
1174	} while (n++ < 5 && !(sts & GSTS_TES(1LL << 31)));
1175	mtx_leave(&iommu->reg_lock);
1176
1177	printf(" set.tes: %d\n", n);
1178
1179	if (n >= 5) {
1180	printf("error.. unable to initialize iommu %d\n",
1181	iommu->id);
1182	iommu->flags \|= IOMMU_FLAGS_BAD0x2;
1183
1184	/* Disable IOMMU */
1185	iommu->gcmd &= ~GCMD_TE(1LL << 31);
1186	mtx_enter(&iommu->reg_lock);
1187	iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1188	mtx_leave(&iommu->reg_lock);
1189
1190	return (1);
1191	}
1192
1193	iommu_flush_ctx(iommu, CTX_GLOBAL, 0, 0, 0);
1194	iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
1195	} else {
1196	iommu->gcmd &= ~GCMD_TE(1LL << 31);
1197
1198	mtx_enter(&iommu->reg_lock);
1199
1200	iommu_write_4(iommu, DMAR_GCMD_REG0x18, iommu->gcmd);
1201	do {
1202	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1203	} while (n++ < 5 && sts & GSTS_TES(1LL << 31));
1204	mtx_leave(&iommu->reg_lock);
1205
1206	printf(" clr.tes: %d\n", n);
1207	}
1208
1209	return (0);
1210	}
1211
1212	/* Intel: Initialize IOMMU */
1213	int
1214	iommu_init(struct acpidmar_softc sc, struct iommu_softc iommu,
1215	struct acpidmar_drhd *dh)
1216	{
1217	static int niommu;
1218	int len = VTD_PAGE_SIZE4096;
1219	int i, gaw;
1220	uint32_t sts;
1221	paddr_t paddr;
1222
1223	if (_bus_space_map(sc->sc_memt, dh->address, len, 0, &iommu->ioh) != 0) {
1224	return (-1);
1225	}
1226
1227	TAILQ_INIT(&iommu->domains)do { (&iommu->domains)->tqh_first = ((void *)0); (& iommu->domains)->tqh_last = &(&iommu->domains )->tqh_first; } while (0);
1228	iommu->id = ++niommu;
1229	iommu->flags = dh->flags;
1230	iommu->segment = dh->segment;
1231	iommu->iot = sc->sc_memt;
1232
1233	iommu->cap = iommu_read_8(iommu, DMAR_CAP_REG0x08);
1234	iommu->ecap = iommu_read_8(iommu, DMAR_ECAP_REG0x10);
1235	iommu->ndoms = cap_nd(iommu->cap)(16 << (((iommu->cap) & 0x7) << 1));
1236
1237	/* Print Capabilities & Extended Capabilities */
1238	DPRINTF(0, " caps: %s%s%s%s%s%s%s%s%s%s%s\n",
1239	iommu->cap & CAP_AFL ? "afl " : "", /* adv fault */
1240	iommu->cap & CAP_RWBF ? "rwbf " : "", /* write-buffer flush */
1241	iommu->cap & CAP_PLMR ? "plmr " : "", /* protected lo region */
1242	iommu->cap & CAP_PHMR ? "phmr " : "", /* protected hi region */
1243	iommu->cap & CAP_CM ? "cm " : "", /* caching mode */
1244	iommu->cap & CAP_ZLR ? "zlr " : "", /* zero-length read */
1245	iommu->cap & CAP_PSI ? "psi " : "", /* page invalidate */
1246	iommu->cap & CAP_DWD ? "dwd " : "", /* write drain */
1247	iommu->cap & CAP_DRD ? "drd " : "", /* read drain */
1248	iommu->cap & CAP_FL1GP ? "Gb " : "", /* 1Gb pages */
1249	iommu->cap & CAP_PI ? "pi " : ""); /* posted interrupts */
1250	DPRINTF(0, " ecap: %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
1251	iommu->ecap & ECAP_C ? "c " : "", /* coherent */
1252	iommu->ecap & ECAP_QI ? "qi " : "", /* queued invalidate */
1253	iommu->ecap & ECAP_DT ? "dt " : "", /* device iotlb */
1254	iommu->ecap & ECAP_IR ? "ir " : "", /* intr remap */
1255	iommu->ecap & ECAP_EIM ? "eim " : "", /* x2apic */
1256	iommu->ecap & ECAP_PT ? "pt " : "", /* passthrough */
1257	iommu->ecap & ECAP_SC ? "sc " : "", /* snoop control */
1258	iommu->ecap & ECAP_ECS ? "ecs " : "", /* extended context */
1259	iommu->ecap & ECAP_MTS ? "mts " : "", /* memory type */
1260	iommu->ecap & ECAP_NEST ? "nest " : "", /* nested translations */
1261	iommu->ecap & ECAP_DIS ? "dis " : "", /* deferred invalidation */
1262	iommu->ecap & ECAP_PASID ? "pas " : "", /* pasid */
1263	iommu->ecap & ECAP_PRS ? "prs " : "", /* page request */
1264	iommu->ecap & ECAP_ERS ? "ers " : "", /* execute request */
1265	iommu->ecap & ECAP_SRS ? "srs " : "", /* supervisor request */
1266	iommu->ecap & ECAP_NWFS ? "nwfs " : "", /* no write flag */
1267	iommu->ecap & ECAP_EAFS ? "eafs " : ""); /* extended accessed flag */
1268
1269	mtx_init(&iommu->reg_lock, IPL_HIGH)do { (void)(((void *)0)); (void)(0); __mtx_init((&iommu-> reg_lock), ((((0xd)) > 0x0 && ((0xd)) < 0x9) ? 0x9 : ((0xd)))); } while (0);
1270
1271	/* Clear Interrupt Masking */
1272	iommu_write_4(iommu, DMAR_FSTS_REG0x34, FSTS_PFO(1LL << 0) \| FSTS_PPF(1LL << 1));
1273
1274	iommu->intr = acpidmar_intr_establish(iommu, IPL_HIGH0xd,
1275	acpidmar_intr, iommu, "dmarintr");
1276
1277	/* Enable interrupts */
1278	sts = iommu_read_4(iommu, DMAR_FECTL_REG0x38);
1279	iommu_write_4(iommu, DMAR_FECTL_REG0x38, sts & ~FECTL_IM(1LL << 31));
1280
1281	/* Allocate root pointer */
1282	iommu->root = iommu_alloc_page(iommu, &paddr);
1283	DPRINTF(0, "Allocated root pointer: pa:%.16llx va:%p\n",
1284	(uint64_t)paddr, iommu->root);
1285	iommu->rtaddr = paddr;
1286	iommu_flush_write_buffer(iommu);
1287	iommu_set_rtaddr(iommu, paddr);
1288
1289	#if 0
1290	if (iommu->ecap & ECAP_QI(1LL << 1)) {
1291	/* Queued Invalidation support */
1292	iommu->qi = iommu_alloc_page(iommu, &iommu->qip);
1293	iommu_write_8(iommu, DMAR_IQT_REG0x88, 0);
1294	iommu_write_8(iommu, DMAR_IQA_REG0x90, iommu->qip \| IQA_QS_2560);
1295	}
1296	if (iommu->ecap & ECAP_IR(1LL << 3)) {
1297	/* Interrupt remapping support */
1298	iommu_write_8(iommu, DMAR_IRTA_REG0xb8, 0);
1299	}
1300	#endif
1301
1302	/* Calculate guest address width and supported guest widths */
1303	gaw = -1;
1304	iommu->mgaw = cap_mgaw(iommu->cap)((uint32_t)(((iommu->cap)>> 16LL) & 0x3F) + 1);
1305	DPRINTF(0, "gaw: %d { ", iommu->mgaw);
1306	for (i = 0; i < 5; i++) {
1307	if (cap_sagaw(iommu->cap)(uint32_t)(((iommu->cap)>> 8LL) & 0x1F) & (1L << i)) {
1308	gaw = VTD_LEVELTOAW(i)(((i) * 9) + 30);
1309	DPRINTF(0, "%d ", gaw);
1310	iommu->agaw = gaw;
1311	}
1312	}
1313	DPRINTF(0, "}\n");
1314
1315	/* Cache current status register bits */
1316	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
1317	if (sts & GSTS_TES(1LL << 31))
1318	iommu->gcmd \|= GCMD_TE(1LL << 31);
1319	if (sts & GSTS_QIES(1LL << 26))
1320	iommu->gcmd \|= GCMD_QIE(1LL << 26);
1321	if (sts & GSTS_IRES(1LL << 25))
1322	iommu->gcmd \|= GCMD_IRE(1LL << 25);
1323	DPRINTF(0, "gcmd: %x preset\n", iommu->gcmd);
1324	acpidmar_intr(iommu);
1325	return (0);
1326	}
1327
1328	/* Read/Write IOMMU register */
1329	uint32_t
1330	iommu_read_4(struct iommu_softc *iommu, int reg)
1331	{
1332	uint32_t v;
1333
1334	v = bus_space_read_4(iommu->iot, iommu->ioh, reg)((iommu->iot)->read_4((iommu->ioh), (reg)));
1335	return (v);
1336	}
1337
1338
1339	void
1340	iommu_write_4(struct iommu_softc *iommu, int reg, uint32_t v)
1341	{
1342	bus_space_write_4(iommu->iot, iommu->ioh, reg, (uint32_t)v)((iommu->iot)->write_4((iommu->ioh), (reg), ((uint32_t )v)));
1343	}
1344
1345	uint64_t
1346	iommu_read_8(struct iommu_softc *iommu, int reg)
1347	{
1348	uint64_t v;
1349
1350	v = bus_space_read_8(iommu->iot, iommu->ioh, reg)((iommu->iot)->read_8((iommu->ioh), (reg)));
1351	return (v);
1352	}
1353
1354	void
1355	iommu_write_8(struct iommu_softc *iommu, int reg, uint64_t v)
1356	{
1357	bus_space_write_8(iommu->iot, iommu->ioh, reg, v)((iommu->iot)->write_8((iommu->ioh), (reg), (v)));
1358	}
1359
1360	/* Check if a device is within a device scope */
1361	int
1362	acpidmar_match_devscope(struct devlist_head *devlist, pci_chipset_tag_t pc,
1363	int sid)
1364	{
1365	struct dmar_devlist *ds;
1366	int sub, sec, i;
1367	int bus, dev, fun, sbus;
1368	pcireg_t reg;
1369	pcitag_t tag;
1370
1371	sbus = sid_bus(sid);
1372	TAILQ_FOREACH(ds, devlist, link)for((ds) = ((devlist)->tqh_first); (ds) != ((void *)0); (ds ) = ((ds)->link.tqe_next)) {
1373	bus = ds->bus;
1374	dev = ds->dp[0].device;
1375	fun = ds->dp[0].function;
1376	/* Walk PCI bridges in path */
1377	for (i = 1; i < ds->ndp; i++) {
1378	tag = pci_make_tag(pc, bus, dev, fun);
1379	reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO0x18);
1380	bus = PPB_BUSINFO_SECONDARY(reg)((reg >> 8) & 0xff);
1381	dev = ds->dp[i].device;
1382	fun = ds->dp[i].function;
1383	}
1384
1385	/* Check for device exact match */
1386	if (sid == mksid(bus, dev, fun)) {
1387	return DMAR_ENDPOINT0x1;
1388	}
1389
1390	/* Check for device subtree match */
1391	if (ds->type == DMAR_BRIDGE0x2) {
1392	tag = pci_make_tag(pc, bus, dev, fun);
1393	reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO0x18);
1394	sec = PPB_BUSINFO_SECONDARY(reg)((reg >> 8) & 0xff);
1395	sub = PPB_BUSINFO_SUBORDINATE(reg)((reg >> 16) & 0xff);
1396	if (sec <= sbus && sbus <= sub) {
1397	return DMAR_BRIDGE0x2;
1398	}
1399	}
1400	}
1401
1402	return (0);
1403	}
1404
1405	struct domain *
1406	domain_create(struct iommu_softc *iommu, int did)
1407	{
1408	struct domain *dom;
1409	int gaw;
1410
1411	DPRINTF(0, "iommu%d: create domain: %.4x\n", iommu->id, did);
1412	dom = malloc(sizeof(*dom), M_DEVBUF2, M_ZERO0x0008 \| M_WAITOK0x0001);
1413	dom->did = did;
1414	dom->iommu = iommu;
1415	dom->pte = iommu_alloc_page(iommu, &dom->ptep);
1416	TAILQ_INIT(&dom->devices)do { (&dom->devices)->tqh_first = ((void *)0); (& dom->devices)->tqh_last = &(&dom->devices)-> tqh_first; } while (0);
1417
1418	/* Setup DMA */
1419	dom->dmat._cookie = dom;
1420	dom->dmat._dmamap_create = dmar_dmamap_create; /* nop */
1421	dom->dmat._dmamap_destroy = dmar_dmamap_destroy; /* nop */
1422	dom->dmat._dmamap_load = dmar_dmamap_load; /* lm */
1423	dom->dmat._dmamap_load_mbuf = dmar_dmamap_load_mbuf; /* lm */
1424	dom->dmat._dmamap_load_uio = dmar_dmamap_load_uio; /* lm */
1425	dom->dmat._dmamap_load_raw = dmar_dmamap_load_raw; /* lm */
1426	dom->dmat._dmamap_unload = dmar_dmamap_unload; /* um */
1427	dom->dmat._dmamap_sync = dmar_dmamap_sync; /* lm */
1428	dom->dmat._dmamem_alloc = dmar_dmamem_alloc; /* nop */
1429	dom->dmat._dmamem_free = dmar_dmamem_free; /* nop */
1430	dom->dmat._dmamem_map = dmar_dmamem_map; /* nop */
1431	dom->dmat._dmamem_unmap = dmar_dmamem_unmap; /* nop */
1432	dom->dmat._dmamem_mmap = dmar_dmamem_mmap;
1433
1434	snprintf(dom->exname, sizeof(dom->exname), "did:%x.%.4x",
1435	iommu->id, dom->did);
1436
1437	/* Setup IOMMU address map */
1438	gaw = min(iommu->agaw, iommu->mgaw);
1439	dom->iovamap = extent_create(dom->exname, 0, (1LL << gaw)-1,
1440	M_DEVBUF2, NULL((void *)0), 0, EX_WAITOK0x0001 \| EX_NOCOALESCE0x0008);
1441
1442	/* Reserve the first 16M */
1443	extent_alloc_region(dom->iovamap, 0, 1610241024, EX_WAITOK0x0001);
1444
1445	/* Zero out MSI Interrupt region */
1446	extent_alloc_region(dom->iovamap, MSI_BASE_ADDRESS0xFEE00000L, MSI_BASE_SIZE0x00100000L,
1447	EX_WAITOK0x0001);
1448	mtx_init(&dom->exlck, IPL_HIGH)do { (void)(((void *)0)); (void)(0); __mtx_init((&dom-> exlck), ((((0xd)) > 0x0 && ((0xd)) < 0x9) ? 0x9 : ((0xd)))); } while (0);
1449
1450	TAILQ_INSERT_TAIL(&iommu->domains, dom, link)do { (dom)->link.tqe_next = ((void )0); (dom)->link.tqe_prev = (&iommu->domains)->tqh_last; (&iommu->domains )->tqh_last = (dom); (&iommu->domains)->tqh_last = &(dom)->link.tqe_next; } while (0);
1451
1452	return dom;
1453	}
1454
1455	void
1456	domain_add_device(struct domain *dom, int sid)
1457	{
1458	struct domain_dev *ddev;
1459
1460	DPRINTF(0, "add %s to iommu%d.%.4x\n", dmar_bdf(sid), dom->iommu->id, dom->did);
1461	ddev = malloc(sizeof(*ddev), M_DEVBUF2, M_ZERO0x0008 \| M_WAITOK0x0001);
1462	ddev->sid = sid;
1463	TAILQ_INSERT_TAIL(&dom->devices, ddev, link)do { (ddev)->link.tqe_next = ((void )0); (ddev)->link. tqe_prev = (&dom->devices)->tqh_last; (&dom-> devices)->tqh_last = (ddev); (&dom->devices)->tqh_last = &(ddev)->link.tqe_next; } while (0);
1464
1465	/* Should set context entry here?? */
1466	}
1467
1468	void
1469	domain_remove_device(struct domain *dom, int sid)
1470	{
1471	struct domain_dev ddev, tmp;
1472
1473	TAILQ_FOREACH_SAFE(ddev, &dom->devices, link, tmp)for ((ddev) = ((&dom->devices)->tqh_first); (ddev) != ((void *)0) && ((tmp) = ((ddev)->link.tqe_next), 1 ); (ddev) = (tmp)) {
1474	if (ddev->sid == sid) {
1475	TAILQ_REMOVE(&dom->devices, ddev, link)do { if (((ddev)->link.tqe_next) != ((void )0)) (ddev)-> link.tqe_next->link.tqe_prev = (ddev)->link.tqe_prev; else (&dom->devices)->tqh_last = (ddev)->link.tqe_prev ; (ddev)->link.tqe_prev = (ddev)->link.tqe_next; ((ddev )->link.tqe_prev) = ((void )-1); ((ddev)->link.tqe_next ) = ((void )-1); } while (0);
1476	free(ddev, sizeof(*ddev), M_DEVBUF2);
1477	}
1478	}
1479	}
1480
1481	/* Lookup domain by segment & source id (bus.device.function) */
1482	struct domain *
1483	domain_lookup(struct acpidmar_softc *sc, int segment, int sid)
1484	{
1485	struct iommu_softc *iommu;
1486	struct domain_dev *ddev;
1487	struct domain *dom;
1488	int rc;
1489
1490	if (sc == NULL((void *)0)) {
1491	return NULL((void *)0);
1492	}
1493
1494	/* Lookup IOMMU for this device */
1495	TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu) != ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
1496	if (iommu->segment != segment)
1497	continue;
1498	/* Check for devscope match or catchall iommu */
1499	rc = acpidmar_match_devscope(&iommu->devices, sc->sc_pc, sid);
1500	if (rc != 0 \|\| iommu->flags) {
1501	break;
1502	}
1503	}
1504	if (!iommu) {
1505	printf("%s: no iommu found\n", dmar_bdf(sid));
1506	return NULL((void *)0);
1507	}
1508
1509	/* Search domain devices */
1510	TAILQ_FOREACH(dom, &iommu->domains, link)for((dom) = ((&iommu->domains)->tqh_first); (dom) != ((void *)0); (dom) = ((dom)->link.tqe_next)) {
1511	TAILQ_FOREACH(ddev, &dom->devices, link)for((ddev) = ((&dom->devices)->tqh_first); (ddev) != ((void *)0); (ddev) = ((ddev)->link.tqe_next)) {
1512	/* XXX: match all functions? */
1513	if (ddev->sid == sid) {
1514	return dom;
1515	}
1516	}
1517	}
1518	if (iommu->ndoms <= 2) {
1519	/* Running out of domains.. create catchall domain */
1520	if (!iommu->unity) {
1521	iommu->unity = domain_create(iommu, 1);
1522	}
1523	dom = iommu->unity;
1524	} else {
1525	dom = domain_create(iommu, --iommu->ndoms);
1526	}
1527	if (!dom) {
1528	printf("no domain here\n");
1529	return NULL((void *)0);
1530	}
1531
1532	/* Add device to domain */
1533	domain_add_device(dom, sid);
1534
1535	return dom;
1536	}
1537
1538	/* Map Guest Pages into IOMMU */
1539	void
1540	_iommu_map(void *dom, vaddr_t va, bus_addr_t gpa, bus_size_t len)
1541	{
1542	bus_size_t i;
1543	paddr_t hpa;
1544
1545	if (dom == NULL((void *)0)) {
1546	return;
1547	}
1548	DPRINTF(1, "Mapping dma: %lx = %lx/%lx\n", va, gpa, len);
1549	for (i = 0; i < len; i += PAGE_SIZE(1 << 12)) {
1550	hpa = 0;
1551	pmap_extract(curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_vmspace->vm_map.pmap, va, &hpa);
1552	domain_map_page(dom, gpa, hpa, PTE_P(1L << 0) \| PTE_R0x00 \| PTE_W(1L << 1));
1553	gpa += PAGE_SIZE(1 << 12);
1554	va += PAGE_SIZE(1 << 12);
1555	}
1556	}
1557
1558	/* Find IOMMU for a given PCI device */
1559	void
1560	_iommu_domain(int segment, int bus, int dev, int func, int id)
1561	{
1562	struct domain *dom;
1563
1564	dom = domain_lookup(acpidmar_sc, segment, mksid(bus, dev, func));
1565	if (dom) {
1566	*id = dom->did;
1567	}
1568	return dom;
1569	}
1570
1571	void
1572	domain_map_device(struct domain *dom, int sid);
1573
1574	void
1575	domain_map_device(struct domain *dom, int sid)
1576	{
1577	struct iommu_softc *iommu;
1578	struct context_entry *ctx;
1579	paddr_t paddr;
1580	int bus, devfn;
1581	int tt, lvl;
1582
1583	iommu = dom->iommu;
1584
1585	bus = sid_bus(sid);
1586	devfn = sid_devfn(sid);
1587	/* AMD attach device */
1588	if (iommu->dte) {
1589	struct ivhd_dte *dte = &iommu->dte[sid];
1590	if (!dte->dw0) {
1591	/* Setup Device Table Entry: bus.devfn */
1592	DPRINTF(1, "@@@ PCI Attach: %.4x[%s] %.4x\n", sid, dmar_bdf(sid), dom->did);
1593	dte_set_host_page_table_root_ptr(dte, dom->ptep);
1594	dte_set_domain(dte, dom->did);
1595	dte_set_mode(dte, 3); /* Set 3 level PTE */
1596	dte_set_tv(dte);
1597	dte_set_valid(dte);
1598	ivhd_flush_devtab(iommu, dom->did);
1599	#ifdef IOMMU_DEBUG
1600	//ivhd_showreg(iommu);
1601	ivhd_showdte(iommu);
1602	#endif
1603	}
1604	return;
1605	}
1606
1607	/* Create Bus mapping */
1608	if (!root_entry_is_valid(&iommu->root[bus])) {
1609	iommu->ctx[bus] = iommu_alloc_page(iommu, &paddr);
1610	iommu->root[bus].lo = paddr \| ROOT_P(1L << 0);
1611	iommu_flush_cache(iommu, &iommu->root[bus],
1612	sizeof(struct root_entry));
1613	DPRINTF(0, "iommu%d: Allocate context for bus: %.2x pa:%.16llx va:%p\n",
1614	iommu->id, bus, (uint64_t)paddr,
1615	iommu->ctx[bus]);
1616	}
1617
1618	/* Create DevFn mapping */
1619	ctx = iommu->ctx[bus] + devfn;
1620	if (!context_entry_is_valid(ctx)) {
1621	tt = CTX_T_MULTI;
1622	lvl = VTD_AWTOLEVEL(iommu->agaw)(((iommu->agaw) - 30) / 9);
1623
1624	/* Initialize context */
1625	context_set_slpte(ctx, dom->ptep);
1626	context_set_translation_type(ctx, tt);
1627	context_set_domain_id(ctx, dom->did);
1628	context_set_address_width(ctx, lvl);
1629	context_set_present(ctx);
1630
1631	/* Flush it */
1632	iommu_flush_cache(iommu, ctx, sizeof(struct context_entry));
1633	if ((iommu->cap & CAP_CM(1LL << 7)) \|\| acpidmar_force_cm) {
1634	iommu_flush_ctx(iommu, CTX_DEVICE, dom->did, sid, 0);
1635	iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
1636	} else {
1637	iommu_flush_write_buffer(iommu);
1638	}
1639	DPRINTF(0, "iommu%d: %s set context ptep:%.16llx lvl:%d did:%.4x tt:%d\n",
1640	iommu->id, dmar_bdf(sid), (uint64_t)dom->ptep, lvl,
1641	dom->did, tt);
1642	}
1643	}
1644
1645	struct domain *
1646	acpidmar_pci_attach(struct acpidmar_softc *sc, int segment, int sid, int mapctx)
1647	{
1648	static struct domain *dom;
1649
1650	dom = domain_lookup(sc, segment, sid);
1651	if (!dom) {
1652	printf("no domain: %s\n", dmar_bdf(sid));
1653	return NULL((void *)0);
1654	}
1655
1656	if (mapctx) {
1657	domain_map_device(dom, sid);
1658	}
1659
1660	return dom;
1661	}
1662
1663	void
1664	acpidmar_pci_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
1665	{
1666	int bus, dev, fun, sid;
1667	struct domain *dom;
1668	pcireg_t reg;
1669
1670	if (!acpidmar_sc) {
1671	/* No DMAR, ignore */
1672	return;
1673	}
1674
1675	/* Add device to our list if valid */
1676	pci_decompose_tag(pc, pa->pa_tag, &bus, &dev, &fun);
1677	sid = mksid(bus, dev, fun);
1678	if (sid_flag[sid] & SID_INVALID0x80000000L)
1679	return;
1680
1681	reg = pci_conf_read(pc, pa->pa_tag, PCI_CLASS_REG0x08);
1682
1683	/* Add device to domain */
1684	dom = acpidmar_pci_attach(acpidmar_sc, pa->pa_domain, sid, 0);
1685	if (dom == NULL((void *)0))
1686	return;
1687
1688	if (PCI_CLASS(reg)(((reg) >> 24) & 0xff) == PCI_CLASS_DISPLAY0x03 &&
1689	PCI_SUBCLASS(reg)(((reg) >> 16) & 0xff) == PCI_SUBCLASS_DISPLAY_VGA0x00) {
1690	dom->flag = DOM_NOMAP0x2;
1691	}
1692	if (PCI_CLASS(reg)(((reg) >> 24) & 0xff) == PCI_CLASS_BRIDGE0x06 &&
1693	PCI_SUBCLASS(reg)(((reg) >> 16) & 0xff) == PCI_SUBCLASS_BRIDGE_ISA0x01) {
1694	/* For ISA Bridges, map 0-16Mb as 1:1 */
1695	printf("dmar: %.4x:%.2x:%.2x.%x mapping ISA\n",
1696	pa->pa_domain, bus, dev, fun);
1697	domain_map_pthru(dom, 0x00, 1610241024);
1698	}
1699
1700	/* Change DMA tag */
1701	pa->pa_dmat = &dom->dmat;
1702	}
1703
1704	/* Create list of device scope entries from ACPI table */
1705	void
1706	acpidmar_parse_devscope(union acpidmar_entry *de, int off, int segment,
1707	struct devlist_head *devlist)
1708	{
1709	struct acpidmar_devscope *ds;
1710	struct dmar_devlist *d;
1711	int dplen, i;
1712
1713	TAILQ_INIT(devlist)do { (devlist)->tqh_first = ((void *)0); (devlist)->tqh_last = &(devlist)->tqh_first; } while (0);
1714	while (off < de->length) {
1715	ds = (struct acpidmar_devscope )((unsigned char )de + off);
1716	off += ds->length;
1717
1718	/* We only care about bridges and endpoints */
1719	if (ds->type != DMAR_ENDPOINT0x1 && ds->type != DMAR_BRIDGE0x2)
1720	continue;
1721
1722	dplen = ds->length - sizeof(*ds);
1723	d = malloc(sizeof(*d) + dplen, M_DEVBUF2, M_ZERO0x0008 \| M_WAITOK0x0001);
1724	d->bus = ds->bus;
1725	d->type = ds->type;
1726	d->ndp = dplen / 2;
1727	d->dp = (void *)&d[1];
1728	memcpy(d->dp, &ds[1], dplen)__builtin_memcpy((d->dp), (&ds[1]), (dplen));
1729	TAILQ_INSERT_TAIL(devlist, d, link)do { (d)->link.tqe_next = ((void )0); (d)->link.tqe_prev = (devlist)->tqh_last; (devlist)->tqh_last = (d); (devlist )->tqh_last = &(d)->link.tqe_next; } while (0);
1730
1731	DPRINTF(1, " %8s %.4x:%.2x.%.2x.%x {",
1732	ds->type == DMAR_BRIDGE ? "bridge" : "endpoint",
1733	segment, ds->bus,
1734	d->dp[0].device,
1735	d->dp[0].function);
1736
1737	for (i = 1; i < d->ndp; i++) {
1738	DPRINTF(1, " %2x.%x ",
1739	d->dp[i].device,
1740	d->dp[i].function);
1741	}
1742	DPRINTF(1, "}\n");
1743	}
1744	}
1745
1746	/* DMA Remapping Hardware Unit */
1747	void
1748	acpidmar_drhd(struct acpidmar_softc sc, union acpidmar_entry de)
1749	{
1750	struct iommu_softc *iommu;
1751
1752	printf("DRHD: segment:%.4x base:%.16llx flags:%.2x\n",
1753	de->drhd.segment,
1754	de->drhd.address,
1755	de->drhd.flags);
1756	iommu = malloc(sizeof(*iommu), M_DEVBUF2, M_ZERO0x0008 \| M_WAITOK0x0001);
1757	acpidmar_parse_devscope(de, sizeof(de->drhd), de->drhd.segment,
1758	&iommu->devices);
1759	iommu_init(sc, iommu, &de->drhd);
1760
1761	if (de->drhd.flags) {
1762	/* Catchall IOMMU goes at end of list */
1763	TAILQ_INSERT_TAIL(&sc->sc_drhds, iommu, link)do { (iommu)->link.tqe_next = ((void )0); (iommu)->link .tqe_prev = (&sc->sc_drhds)->tqh_last; (&sc-> sc_drhds)->tqh_last = (iommu); (&sc->sc_drhds)-> tqh_last = &(iommu)->link.tqe_next; } while (0);
1764	} else {
1765	TAILQ_INSERT_HEAD(&sc->sc_drhds, iommu, link)do { if (((iommu)->link.tqe_next = (&sc->sc_drhds)-> tqh_first) != ((void *)0)) (&sc->sc_drhds)->tqh_first ->link.tqe_prev = &(iommu)->link.tqe_next; else (& sc->sc_drhds)->tqh_last = &(iommu)->link.tqe_next ; (&sc->sc_drhds)->tqh_first = (iommu); (iommu)-> link.tqe_prev = &(&sc->sc_drhds)->tqh_first; } while (0);
1766	}
1767	}
1768
1769	/* Reserved Memory Region Reporting */
1770	void
1771	acpidmar_rmrr(struct acpidmar_softc sc, union acpidmar_entry de)
1772	{
1773	struct rmrr_softc *rmrr;
1774	bios_memmap_t im, jm;
1775	uint64_t start, end;
1776
1777	printf("RMRR: segment:%.4x range:%.16llx-%.16llx\n",
1778	de->rmrr.segment, de->rmrr.base, de->rmrr.limit);
1779	if (de->rmrr.limit <= de->rmrr.base) {
1780	printf(" buggy BIOS\n");
1781	return;
1782	}
1783
1784	rmrr = malloc(sizeof(*rmrr), M_DEVBUF2, M_ZERO0x0008 \| M_WAITOK0x0001);
1785	rmrr->start = trunc_page(de->rmrr.base)((de->rmrr.base) & ~((1 << 12) - 1));
1786	rmrr->end = round_page(de->rmrr.limit)(((de->rmrr.limit) + ((1 << 12) - 1)) & ~((1 << 12) - 1));
1787	rmrr->segment = de->rmrr.segment;
1788	acpidmar_parse_devscope(de, sizeof(de->rmrr), de->rmrr.segment,
1789	&rmrr->devices);
1790
1791	for (im = bios_memmap; im->type != BIOS_MAP_END0x00; im++) {
1792	if (im->type != BIOS_MAP_RES0x02)
1793	continue;
1794	/* Search for adjacent reserved regions */
1795	start = im->addr;
1796	end = im->addr+im->size;
1797	for (jm = im+1; jm->type == BIOS_MAP_RES0x02 && end == jm->addr;
1798	jm++) {
1799	end = jm->addr+jm->size;
1800	}
1801	printf("e820: %.16llx - %.16llx\n", start, end);
1802	if (start <= rmrr->start && rmrr->end <= end) {
1803	/* Bah.. some buggy BIOS stomp outside RMRR */
1804	printf(" ** inside E820 Reserved %.16llx %.16llx\n",
1805	start, end);
1806	rmrr->start = trunc_page(start)((start) & ~((1 << 12) - 1));
1807	rmrr->end = round_page(end)(((end) + ((1 << 12) - 1)) & ~((1 << 12) - 1) );
1808	break;
1809	}
1810	}
1811	TAILQ_INSERT_TAIL(&sc->sc_rmrrs, rmrr, link)do { (rmrr)->link.tqe_next = ((void )0); (rmrr)->link. tqe_prev = (&sc->sc_rmrrs)->tqh_last; (&sc-> sc_rmrrs)->tqh_last = (rmrr); (&sc->sc_rmrrs)->tqh_last = &(rmrr)->link.tqe_next; } while (0);
1812	}
1813
1814	/* Root Port ATS Reporting */
1815	void
1816	acpidmar_atsr(struct acpidmar_softc sc, union acpidmar_entry de)
1817	{
1818	struct atsr_softc *atsr;
1819
1820	printf("ATSR: segment:%.4x flags:%x\n",
1821	de->atsr.segment,
1822	de->atsr.flags);
1823
1824	atsr = malloc(sizeof(*atsr), M_DEVBUF2, M_ZERO0x0008 \| M_WAITOK0x0001);
1825	atsr->flags = de->atsr.flags;
1826	atsr->segment = de->atsr.segment;
1827	acpidmar_parse_devscope(de, sizeof(de->atsr), de->atsr.segment,
1828	&atsr->devices);
1829
1830	TAILQ_INSERT_TAIL(&sc->sc_atsrs, atsr, link)do { (atsr)->link.tqe_next = ((void )0); (atsr)->link. tqe_prev = (&sc->sc_atsrs)->tqh_last; (&sc-> sc_atsrs)->tqh_last = (atsr); (&sc->sc_atsrs)->tqh_last = &(atsr)->link.tqe_next; } while (0);
1831	}
1832
1833	void
1834	acpidmar_init(struct acpidmar_softc sc, struct acpi_dmar dmar)
1835	{
1836	struct rmrr_softc *rmrr;
1837	struct iommu_softc *iommu;
1838	struct domain *dom;
1839	struct dmar_devlist *dl;
1840	union acpidmar_entry *de;
1841	int off, sid, rc;
1842
1843	domain_map_page = domain_map_page_intel;
1844	printf(": hardware width: %d, intr_remap:%d x2apic_opt_out:%d\n",
1845	dmar->haw+1,
1846	!!(dmar->flags & 0x1),
1847	!!(dmar->flags & 0x2));
1848	sc->sc_haw = dmar->haw+1;
1849	sc->sc_flags = dmar->flags;
1850
1851	TAILQ_INIT(&sc->sc_drhds)do { (&sc->sc_drhds)->tqh_first = ((void *)0); (& sc->sc_drhds)->tqh_last = &(&sc->sc_drhds)-> tqh_first; } while (0);
1852	TAILQ_INIT(&sc->sc_rmrrs)do { (&sc->sc_rmrrs)->tqh_first = ((void *)0); (& sc->sc_rmrrs)->tqh_last = &(&sc->sc_rmrrs)-> tqh_first; } while (0);
1853	TAILQ_INIT(&sc->sc_atsrs)do { (&sc->sc_atsrs)->tqh_first = ((void *)0); (& sc->sc_atsrs)->tqh_last = &(&sc->sc_atsrs)-> tqh_first; } while (0);
1854
1855	off = sizeof(*dmar);
1856	while (off < dmar->hdr.length) {
1857	de = (union acpidmar_entry )((unsigned char )dmar + off);
1858	switch (de->type) {
1859	case DMAR_DRHD0x0:
1860	acpidmar_drhd(sc, de);
1861	break;
1862	case DMAR_RMRR0x1:
1863	acpidmar_rmrr(sc, de);
1864	break;
1865	case DMAR_ATSR0x2:
1866	acpidmar_atsr(sc, de);
1867	break;
1868	default:
1869	printf("DMAR: unknown %x\n", de->type);
1870	break;
1871	}
1872	off += de->length;
1873	}
1874
1875	/* Pre-create domains for iommu devices */
1876	TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu) != ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
1877	TAILQ_FOREACH(dl, &iommu->devices, link)for((dl) = ((&iommu->devices)->tqh_first); (dl) != ( (void *)0); (dl) = ((dl)->link.tqe_next)) {
1878	sid = mksid(dl->bus, dl->dp[0].device,
1879	dl->dp[0].function);
1880	dom = acpidmar_pci_attach(sc, iommu->segment, sid, 0);
1881	if (dom != NULL((void *)0)) {
1882	printf("%.4x:%.2x:%.2x.%x iommu:%d did:%.4x\n",
1883	iommu->segment, dl->bus, dl->dp[0].device, dl->dp[0].function,
1884	iommu->id, dom->did);
1885	}
1886	}
1887	}
1888	/* Map passthrough pages for RMRR */
1889	TAILQ_FOREACH(rmrr, &sc->sc_rmrrs, link)for((rmrr) = ((&sc->sc_rmrrs)->tqh_first); (rmrr) != ((void *)0); (rmrr) = ((rmrr)->link.tqe_next)) {
1890	TAILQ_FOREACH(dl, &rmrr->devices, link)for((dl) = ((&rmrr->devices)->tqh_first); (dl) != ( (void *)0); (dl) = ((dl)->link.tqe_next)) {
1891	sid = mksid(dl->bus, dl->dp[0].device,
1892	dl->dp[0].function);
1893	dom = acpidmar_pci_attach(sc, rmrr->segment, sid, 0);
1894	if (dom != NULL((void *)0)) {
1895	printf("%s map ident: %.16llx %.16llx\n",
1896	dom_bdf(dom), rmrr->start, rmrr->end);
1897	domain_map_pthru(dom, rmrr->start, rmrr->end);
1898	rc = extent_alloc_region(dom->iovamap,
	Value stored to 'rc' is never read
1899	rmrr->start, rmrr->end,
1900	EX_WAITOK0x0001 \| EX_CONFLICTOK0x0080);
1901	}
1902	}
1903	}
1904	}
1905
1906
1907	/*=====================================================
1908	* AMD Vi
1909	=====================================================/
1910	void acpiivrs_ivhd(struct acpidmar_softc , struct acpi_ivhd );
1911	int ivhd_iommu_init(struct acpidmar_softc , struct iommu_softc ,
1912	struct acpi_ivhd *);
1913	int _ivhd_issue_command(struct iommu_softc , const struct ivhd_command );
1914	void ivhd_show_event(struct iommu_softc , struct ivhd_event evt, int);
1915	int ivhd_issue_command(struct iommu_softc , const struct ivhd_command , int);
1916	int ivhd_invalidate_domain(struct iommu_softc *, int);
1917	void ivhd_intr_map(struct iommu_softc *, int);
1918	void ivhd_checkerr(struct iommu_softc *iommu);
1919	int acpiivhd_intr(void *);
1920
1921	int
1922	acpiivhd_intr(void *ctx)
1923	{
1924	struct iommu_softc *iommu = ctx;
1925
1926	if (!iommu->dte)
1927	return (0);
1928	ivhd_poll_events(iommu);
1929	return (1);
1930	}
1931
1932	/* Setup interrupt for AMD */
1933	void
1934	ivhd_intr_map(struct iommu_softc *iommu, int devid) {
1935	pci_intr_handle_t ih;
1936
1937	if (iommu->intr)
1938	return;
1939	ih.tag = pci_make_tag(NULL((void *)0), sid_bus(devid), sid_dev(devid), sid_fun(devid));
1940	ih.line = APIC_INT_VIA_MSG0x20000000;
1941	ih.pin = 0;
1942	iommu->intr = pci_intr_establish(NULL((void *)0), ih, IPL_NET0x4 \| IPL_MPSAFE0x100,
1943	acpiivhd_intr, iommu, "amd_iommu");
1944	printf("amd iommu intr: %p\n", iommu->intr);
1945	}
1946
1947	void
1948	_dumppte(struct pte_entry *pte, int lvl, vaddr_t va)
1949	{
1950	char *pfx[] = { " ", " ", " ", " ", "" };
1951	uint64_t i, sh;
1952	struct pte_entry *npte;
1953
1954	for (i = 0; i < 512; i++) {
1955	sh = (i << (((lvl-1) * 9) + 12));
1956	if (pte[i].val & PTE_P(1L << 0)) {
1957	if (lvl > 1) {
1958	npte = (void )PMAP_DIRECT_MAP((pte[i].val & PTE_PADDR_MASK))((vaddr_t)(((((511 - 4) (1ULL << 39))) \| 0xffff000000000000 )) + ((pte[i].val & 0x000FFFFFFFFFF000LL)));
1959	printf("%slvl%d: %.16llx nxt:%llu\n", pfx[lvl], lvl,
1960	pte[i].val, (pte[i].val >> 9) & 7);
1961	_dumppte(npte, lvl-1, va \| sh);
1962	} else {
1963	printf("%slvl%d: %.16llx <- %.16llx \n", pfx[lvl], lvl,
1964	pte[i].val, va \| sh);
1965	}
1966	}
1967	}
1968	}
1969
1970	void
1971	ivhd_showpage(struct iommu_softc *iommu, int sid, paddr_t paddr)
1972	{
1973	struct domain *dom;
1974	static int show = 0;
1975
1976	if (show > 10)
1977	return;
1978	show++;
1979	dom = acpidmar_pci_attach(acpidmar_sc, 0, sid, 0);
1980	if (!dom)
1981	return;
1982	printf("DTE: %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
1983	iommu->dte[sid].dw0,
1984	iommu->dte[sid].dw1,
1985	iommu->dte[sid].dw2,
1986	iommu->dte[sid].dw3,
1987	iommu->dte[sid].dw4,
1988	iommu->dte[sid].dw5,
1989	iommu->dte[sid].dw6,
1990	iommu->dte[sid].dw7);
1991	_dumppte(dom->pte, 3, 0);
1992	}
1993
1994	/* Display AMD IOMMU Error */
1995	void
1996	ivhd_show_event(struct iommu_softc iommu, struct ivhd_event evt, int head)
1997	{
1998	int type, sid, did, flag;
1999	uint64_t address;
2000
2001	/* Get Device, Domain, Address and Type of event */
2002	sid = __EXTRACT(evt->dw0, EVT_SID)(((evt->dw0) >> 0) & 0xFFFF);
2003	type = __EXTRACT(evt->dw1, EVT_TYPE)(((evt->dw1) >> 28) & 0xF);
2004	did = __EXTRACT(evt->dw1, EVT_DID)(((evt->dw1) >> 0) & 0xFFFF);
2005	flag = __EXTRACT(evt->dw1, EVT_FLAG)(((evt->dw1) >> 16) & 0xFFF);
2006	address = _get64(&evt->dw2)(uint64_t )(&evt->dw2);
2007
2008	printf("=== IOMMU Error[%.4x]: ", head);
2009	switch (type) {
2010	case ILLEGAL_DEV_TABLE_ENTRY:
2011	printf("illegal dev table entry dev=%s addr=0x%.16llx %s, %s, %s, %s\n",
2012	dmar_bdf(sid), address,
2013	evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2014	evt->dw1 & EVT_RZ(1L << 23) ? "reserved bit" : "invalid level",
2015	evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2016	evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2017	ivhd_showdte(iommu);
2018	break;
2019	case IO_PAGE_FAULT:
2020	printf("io page fault dev=%s did=0x%.4x addr=0x%.16llx\n%s, %s, %s, %s, %s, %s\n",
2021	dmar_bdf(sid), did, address,
2022	evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2023	evt->dw1 & EVT_RZ(1L << 23) ? "reserved bit" : "invalid level",
2024	evt->dw1 & EVT_PE(1L << 22) ? "no perm" : "perm",
2025	evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2026	evt->dw1 & EVT_PR(1L << 20) ? "present" : "not present",
2027	evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2028	ivhd_showdte(iommu);
2029	ivhd_showpage(iommu, sid, address);
2030	break;
2031	case DEV_TAB_HARDWARE_ERROR:
2032	printf("device table hardware error dev=%s addr=0x%.16llx %s, %s, %s\n",
2033	dmar_bdf(sid), address,
2034	evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2035	evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2036	evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2037	ivhd_showdte(iommu);
2038	break;
2039	case PAGE_TAB_HARDWARE_ERROR:
2040	printf("page table hardware error dev=%s addr=0x%.16llx %s, %s, %s\n",
2041	dmar_bdf(sid), address,
2042	evt->dw1 & EVT_TR(1L << 24) ? "translation" : "transaction",
2043	evt->dw1 & EVT_RW(1L << 21) ? "write" : "read",
2044	evt->dw1 & EVT_I(1L << 19) ? "interrupt" : "memory");
2045	ivhd_showdte(iommu);
2046	break;
2047	case ILLEGAL_COMMAND_ERROR:
2048	printf("illegal command addr=0x%.16llx\n", address);
2049	ivhd_showcmd(iommu);
2050	break;
2051	case COMMAND_HARDWARE_ERROR:
2052	printf("command hardware error addr=0x%.16llx flag=0x%.4x\n",
2053	address, flag);
2054	ivhd_showcmd(iommu);
2055	break;
2056	case IOTLB_INV_TIMEOUT:
2057	printf("iotlb invalidation timeout dev=%s address=0x%.16llx\n",
2058	dmar_bdf(sid), address);
2059	break;
2060	case INVALID_DEVICE_REQUEST:
2061	printf("invalid device request dev=%s addr=0x%.16llx flag=0x%.4x\n",
2062	dmar_bdf(sid), address, flag);
2063	break;
2064	default:
2065	printf("unknown type=0x%.2x\n", type);
2066	break;
2067	}
2068	/* Clear old event */
2069	evt->dw0 = 0;
2070	evt->dw1 = 0;
2071	evt->dw2 = 0;
2072	evt->dw3 = 0;
2073	}
2074
2075	/* AMD: Process IOMMU error from hardware */
2076	int
2077	ivhd_poll_events(struct iommu_softc *iommu)
2078	{
2079	uint32_t head, tail;
2080	int sz;
2081
2082	sz = sizeof(struct ivhd_event);
2083	head = iommu_read_4(iommu, EVT_HEAD_REG0x2010);
2084	tail = iommu_read_4(iommu, EVT_TAIL_REG0x2018);
2085	if (head == tail) {
2086	/* No pending events */
2087	return (0);
2088	}
2089	while (head != tail) {
2090	ivhd_show_event(iommu, iommu->evt_tbl + head, head);
2091	head = (head + sz) % EVT_TBL_SIZE4096;
2092	}
2093	iommu_write_4(iommu, EVT_HEAD_REG0x2010, head);
2094	return (0);
2095	}
2096
2097	/* AMD: Issue command to IOMMU queue */
2098	int
2099	_ivhd_issue_command(struct iommu_softc iommu, const struct ivhd_command cmd)
2100	{
2101	u_long rf;
2102	uint32_t head, tail, next;
2103	int sz;
2104
2105	head = iommu_read_4(iommu, CMD_HEAD_REG0x2000);
2106	sz = sizeof(*cmd);
2107	rf = intr_disable();
2108	tail = iommu_read_4(iommu, CMD_TAIL_REG0x2008);
2109	next = (tail + sz) % CMD_TBL_SIZE4096;
2110	if (next == head) {
2111	printf("FULL\n");
2112	/* Queue is full */
2113	intr_restore(rf);
2114	return -EBUSY16;
2115	}
2116	memcpy(iommu->cmd_tbl + tail, cmd, sz)__builtin_memcpy((iommu->cmd_tbl + tail), (cmd), (sz));
2117	iommu_write_4(iommu, CMD_TAIL_REG0x2008, next);
2118	intr_restore(rf);
2119	return (tail / sz);
2120	}
2121
2122	#define IVHD_MAXDELAY8 8
2123
2124	int
2125	ivhd_issue_command(struct iommu_softc iommu, const struct ivhd_command cmd, int wait)
2126	{
2127	struct ivhd_command wq = { 0 };
2128	volatile uint64_t wv __aligned(16)__attribute__((__aligned__(16))) = 0LL;
2129	paddr_t paddr;
2130	int rc, i;
2131
2132	rc = _ivhd_issue_command(iommu, cmd);
2133	if (rc >= 0 && wait) {
2134	/* Wait for previous commands to complete.
2135	* Store address of completion variable to command */
2136	pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)&wv, &paddr);
2137	wq.dw0 = (paddr & ~0xF) \| 0x1;
2138	wq.dw1 = (COMPLETION_WAIT << CMD_SHIFT28) \| ((paddr >> 32) & 0xFFFFF);
2139	wq.dw2 = 0xDEADBEEF;
2140	wq.dw3 = 0xFEEDC0DE;
2141
2142	rc = _ivhd_issue_command(iommu, &wq);
2143	/* wv will change to value in dw2/dw3 when command is complete */
2144	for (i = 0; i < IVHD_MAXDELAY8 && !wv; i++) {
2145	DELAY(10 << i)(*delay_func)(10 << i);
2146	}
2147	if (i == IVHD_MAXDELAY8) {
2148	printf("ivhd command timeout: %.8x %.8x %.8x %.8x wv:%llx idx:%x\n",
2149	cmd->dw0, cmd->dw1, cmd->dw2, cmd->dw3, wv, rc);
2150	}
2151	}
2152	return rc;
2153
2154	}
2155
2156	/* AMD: Flush changes to Device Table Entry for a specific domain */
2157	int
2158	ivhd_flush_devtab(struct iommu_softc *iommu, int did)
2159	{
2160	struct ivhd_command cmd = {
2161	.dw0 = did,
2162	.dw1 = INVALIDATE_DEVTAB_ENTRY << CMD_SHIFT28
2163	};
2164
2165	return ivhd_issue_command(iommu, &cmd, 1);
2166	}
2167
2168	/* AMD: Invalidate all IOMMU device and page tables */
2169	int
2170	ivhd_invalidate_iommu_all(struct iommu_softc *iommu)
2171	{
2172	struct ivhd_command cmd = {
2173	.dw1 = INVALIDATE_IOMMU_ALL << CMD_SHIFT28
2174	};
2175
2176	return ivhd_issue_command(iommu, &cmd, 0);
2177	}
2178
2179	/* AMD: Invalidate interrupt remapping */
2180	int
2181	ivhd_invalidate_interrupt_table(struct iommu_softc *iommu, int did)
2182	{
2183	struct ivhd_command cmd = {
2184	.dw0 = did,
2185	.dw1 = INVALIDATE_INTERRUPT_TABLE << CMD_SHIFT28
2186	};
2187
2188	return ivhd_issue_command(iommu, &cmd, 0);
2189	}
2190
2191	/* AMD: Invalidate all page tables in a domain */
2192	int
2193	ivhd_invalidate_domain(struct iommu_softc *iommu, int did)
2194	{
2195	struct ivhd_command cmd = { .dw1 = did \| (INVALIDATE_IOMMU_PAGES << CMD_SHIFT28) };
2196
2197	cmd.dw2 = 0xFFFFF000 \| 0x3;
2198	cmd.dw3 = 0x7FFFFFFF;
2199	return ivhd_issue_command(iommu, &cmd, 1);
2200	}
2201
2202	/* AMD: Display Registers */
2203	void
2204	ivhd_showreg(struct iommu_softc *iommu)
2205	{
2206	printf("---- dt:%.16llx cmd:%.16llx evt:%.16llx ctl:%.16llx sts:%.16llx\n",
2207	iommu_read_8(iommu, DEV_TAB_BASE_REG0x0000),
2208	iommu_read_8(iommu, CMD_BASE_REG0x0008),
2209	iommu_read_8(iommu, EVT_BASE_REG0x0010),
2210	iommu_read_8(iommu, IOMMUCTL_REG0x0018),
2211	iommu_read_8(iommu, IOMMUSTS_REG0x2020));
2212	printf("---- cmd queue:%.16llx %.16llx evt queue:%.16llx %.16llx\n",
2213	iommu_read_8(iommu, CMD_HEAD_REG0x2000),
2214	iommu_read_8(iommu, CMD_TAIL_REG0x2008),
2215	iommu_read_8(iommu, EVT_HEAD_REG0x2010),
2216	iommu_read_8(iommu, EVT_TAIL_REG0x2018));
2217	}
2218
2219	/* AMD: Generate Errors to test event handler */
2220	void
2221	ivhd_checkerr(struct iommu_softc *iommu)
2222	{
2223	struct ivhd_command cmd = { -1, -1, -1, -1 };
2224
2225	/* Generate ILLEGAL DEV TAB entry? */
2226	iommu->dte[0x2303].dw0 = -1; /* invalid */
2227	iommu->dte[0x2303].dw2 = 0x1234; /* domain */
2228	iommu->dte[0x2303].dw7 = -1; /* reserved */
2229	ivhd_flush_devtab(iommu, 0x1234);
2230	ivhd_poll_events(iommu);
2231
2232	/* Generate ILLEGAL_COMMAND_ERROR : ok */
2233	ivhd_issue_command(iommu, &cmd, 0);
2234	ivhd_poll_events(iommu);
2235
2236	/* Generate page hardware error */
2237	}
2238
2239	/* AMD: Show Device Table Entry */
2240	void
2241	ivhd_showdte(struct iommu_softc *iommu)
2242	{
2243	int i;
2244
2245	for (i = 0; i < 65536; i++) {
2246	if (iommu->dte[i].dw0) {
2247	printf("%.2x:%.2x.%x: %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
2248	i >> 8, (i >> 3) & 0x1F, i & 0x7,
2249	iommu->dte[i].dw0, iommu->dte[i].dw1,
2250	iommu->dte[i].dw2, iommu->dte[i].dw3,
2251	iommu->dte[i].dw4, iommu->dte[i].dw5,
2252	iommu->dte[i].dw6, iommu->dte[i].dw7);
2253	}
2254	}
2255	}
2256
2257	/* AMD: Show command entries */
2258	void
2259	ivhd_showcmd(struct iommu_softc *iommu)
2260	{
2261	struct ivhd_command *ihd;
2262	paddr_t phd;
2263	int i;
2264
2265	ihd = iommu->cmd_tbl;
2266	phd = iommu_read_8(iommu, CMD_BASE_REG0x0008) & CMD_BASE_MASK0x000FFFFFFFFFF000LL;
2267	for (i = 0; i < 4096 / 128; i++) {
2268	printf("%.2x: %.16llx %.8x %.8x %.8x %.8x\n", i,
2269	(uint64_t)phd + i * sizeof(*ihd),
2270	ihd[i].dw0,ihd[i].dw1,ihd[i].dw2,ihd[i].dw3);
2271	}
2272	}
2273
2274	#define _c(x)(int)((iommu->ecap >> x_SHIFT) & x_MASK) (int)((iommu->ecap >> x ##_SHIFT) & x ## _MASK)
2275
2276	/* AMD: Initialize IOMMU */
2277	int
2278	ivhd_iommu_init(struct acpidmar_softc sc, struct iommu_softc iommu,
2279	struct acpi_ivhd *ivhd)
2280	{
2281	static int niommu;
2282	paddr_t paddr;
2283	uint64_t ov;
2284
2285	if (sc == NULL((void )0) \|\| iommu == NULL((void )0) \|\| ivhd == NULL((void *)0)) {
2286	printf("Bad pointer to iommu_init!\n");
2287	return -1;
2288	}
2289	if (_bus_space_map(sc->sc_memt, ivhd->address, 0x80000, 0, &iommu->ioh) != 0) {
2290	printf("Bus Space Map fails\n");
2291	return -1;
2292	}
2293	TAILQ_INIT(&iommu->domains)do { (&iommu->domains)->tqh_first = ((void *)0); (& iommu->domains)->tqh_last = &(&iommu->domains )->tqh_first; } while (0);
2294	TAILQ_INIT(&iommu->devices)do { (&iommu->devices)->tqh_first = ((void *)0); (& iommu->devices)->tqh_last = &(&iommu->devices )->tqh_first; } while (0);
2295
2296	/* Setup address width and number of domains */
2297	iommu->id = ++niommu;
2298	iommu->iot = sc->sc_memt;
2299	iommu->mgaw = 48;
2300	iommu->agaw = 48;
2301	iommu->flags = 1;
2302	iommu->segment = 0;
2303	iommu->ndoms = 256;
2304
2305	printf(": AMD iommu%d at 0x%.8llx\n", iommu->id, ivhd->address);
2306
2307	iommu->ecap = iommu_read_8(iommu, EXTFEAT_REG0x0030);
2308	DPRINTF(0,"iommu%d: ecap:%.16llx ", iommu->id, iommu->ecap);
2309	DPRINTF(0,"%s%s%s%s%s%s%s%s\n",
2310	iommu->ecap & EFR_PREFSUP ? "pref " : "",
2311	iommu->ecap & EFR_PPRSUP ? "ppr " : "",
2312	iommu->ecap & EFR_NXSUP ? "nx " : "",
2313	iommu->ecap & EFR_GTSUP ? "gt " : "",
2314	iommu->ecap & EFR_IASUP ? "ia " : "",
2315	iommu->ecap & EFR_GASUP ? "ga " : "",
2316	iommu->ecap & EFR_HESUP ? "he " : "",
2317	iommu->ecap & EFR_PCSUP ? "pc " : "");
2318	DPRINTF(0,"hats:%x gats:%x glxsup:%x smif:%x smifrc:%x gam:%x\n",
2319	_c(EFR_HATS), _c(EFR_GATS), _c(EFR_GLXSUP), _c(EFR_SMIFSUP),
2320	_c(EFR_SMIFRC), _c(EFR_GAMSUP));
2321
2322	/* Turn off iommu */
2323	ov = iommu_read_8(iommu, IOMMUCTL_REG0x0018);
2324	iommu_write_8(iommu, IOMMUCTL_REG0x0018, ov & ~(CTL_IOMMUEN(1L << 0) \| CTL_COHERENT(1L << 10) \|
2325	CTL_HTTUNEN(1L << 1) \| CTL_RESPASSPW(1L << 9) \| CTL_PASSPW(1L << 8) \| CTL_ISOC(1L << 11)));
2326
2327	/* Enable intr, mark IOMMU device as invalid for remap */
2328	sid_flag[ivhd->devid] \|= SID_INVALID0x80000000L;
2329	ivhd_intr_map(iommu, ivhd->devid);
2330
2331	/* Setup command buffer with 4k buffer (128 entries) */
2332	iommu->cmd_tbl = iommu_alloc_page(iommu, &paddr);
2333	iommu_write_8(iommu, CMD_BASE_REG0x0008, (paddr & CMD_BASE_MASK0x000FFFFFFFFFF000LL) \| CMD_TBL_LEN_4K(8LL << 56));
2334	iommu_write_4(iommu, CMD_HEAD_REG0x2000, 0x00);
2335	iommu_write_4(iommu, CMD_TAIL_REG0x2008, 0x00);
2336	iommu->cmd_tblp = paddr;
2337
2338	/* Setup event log with 4k buffer (128 entries) */
2339	iommu->evt_tbl = iommu_alloc_page(iommu, &paddr);
2340	iommu_write_8(iommu, EVT_BASE_REG0x0010, (paddr & EVT_BASE_MASK0x000FFFFFFFFFF000LL) \| EVT_TBL_LEN_4K(8LL << 56));
2341	iommu_write_4(iommu, EVT_HEAD_REG0x2010, 0x00);
2342	iommu_write_4(iommu, EVT_TAIL_REG0x2018, 0x00);
2343	iommu->evt_tblp = paddr;
2344
2345	/* Setup device table
2346	* 1 entry per source ID (bus:device:function - 64k entries)
2347	*/
2348	iommu->dte = sc->sc_hwdte;
2349	iommu_write_8(iommu, DEV_TAB_BASE_REG0x0000, (sc->sc_hwdtep & DEV_TAB_MASK0x000FFFFFFFFFF000LL) \| DEV_TAB_LEN0x1FF);
2350
2351	/* Enable IOMMU */
2352	ov \|= (CTL_IOMMUEN(1L << 0) \| CTL_EVENTLOGEN(1L << 2) \| CTL_CMDBUFEN(1L << 12) \| CTL_EVENTINTEN(1L << 3));
2353	if (ivhd->flags & IVHD_COHERENT(1L << 5))
2354	ov \|= CTL_COHERENT(1L << 10);
2355	if (ivhd->flags & IVHD_HTTUNEN(1L << 0))
2356	ov \|= CTL_HTTUNEN(1L << 1);
2357	if (ivhd->flags & IVHD_RESPASSPW(1L << 2))
2358	ov \|= CTL_RESPASSPW(1L << 9);
2359	if (ivhd->flags & IVHD_PASSPW(1L << 1))
2360	ov \|= CTL_PASSPW(1L << 8);
2361	if (ivhd->flags & IVHD_ISOC(1L << 3))
2362	ov \|= CTL_ISOC(1L << 11);
2363	ov &= ~(CTL_INVTIMEOUT_MASK0x7 << CTL_INVTIMEOUT_SHIFT5);
2364	ov \|= (CTL_INVTIMEOUT_10MS2 << CTL_INVTIMEOUT_SHIFT5);
2365	iommu_write_8(iommu, IOMMUCTL_REG0x0018, ov);
2366
2367	ivhd_invalidate_iommu_all(iommu);
2368
2369	TAILQ_INSERT_TAIL(&sc->sc_drhds, iommu, link)do { (iommu)->link.tqe_next = ((void )0); (iommu)->link .tqe_prev = (&sc->sc_drhds)->tqh_last; (&sc-> sc_drhds)->tqh_last = (iommu); (&sc->sc_drhds)-> tqh_last = &(iommu)->link.tqe_next; } while (0);
2370	return 0;
2371	}
2372
2373	void
2374	acpiivrs_ivhd(struct acpidmar_softc sc, struct acpi_ivhd ivhd)
2375	{
2376	struct iommu_softc *iommu;
2377	struct acpi_ivhd_ext *ext;
2378	union acpi_ivhd_entry *ie;
2379	int start, off, dte, all_dte = 0;
2380
2381	if (ivhd->type == IVRS_IVHD_EXT0x11) {
2382	ext = (struct acpi_ivhd_ext *)ivhd;
2383	DPRINTF(0,"ivhd: %.2x %.2x %.4x %.4x:%s %.4x %.16llx %.4x %.8x %.16llx\n",
2384	ext->type, ext->flags, ext->length,
2385	ext->segment, dmar_bdf(ext->devid), ext->cap,
2386	ext->address, ext->info,
2387	ext->attrib, ext->efr);
2388	if (ext->flags & IVHD_PPRSUP(1L << 7))
2389	DPRINTF(0," PPRSup");
2390	if (ext->flags & IVHD_PREFSUP(1L << 6))
2391	DPRINTF(0," PreFSup");
2392	if (ext->flags & IVHD_COHERENT(1L << 5))
2393	DPRINTF(0," Coherent");
2394	if (ext->flags & IVHD_IOTLB(1L << 4))
2395	DPRINTF(0," Iotlb");
2396	if (ext->flags & IVHD_ISOC(1L << 3))
2397	DPRINTF(0," ISoc");
2398	if (ext->flags & IVHD_RESPASSPW(1L << 2))
2399	DPRINTF(0," ResPassPW");
2400	if (ext->flags & IVHD_PASSPW(1L << 1))
2401	DPRINTF(0," PassPW");
2402	if (ext->flags & IVHD_HTTUNEN(1L << 0))
2403	DPRINTF(0, " HtTunEn");
2404	if (ext->flags)
2405	DPRINTF(0,"\n");
2406	off = sizeof(*ext);
2407	iommu = malloc(sizeof(*iommu), M_DEVBUF2, M_ZERO0x0008\|M_WAITOK0x0001);
2408	ivhd_iommu_init(sc, iommu, ivhd);
2409	} else {
2410	DPRINTF(0,"ivhd: %.2x %.2x %.4x %.4x:%s %.4x %.16llx %.4x %.8x\n",
2411	ivhd->type, ivhd->flags, ivhd->length,
2412	ivhd->segment, dmar_bdf(ivhd->devid), ivhd->cap,
2413	ivhd->address, ivhd->info,
2414	ivhd->feature);
2415	if (ivhd->flags & IVHD_PPRSUP(1L << 7))
2416	DPRINTF(0," PPRSup");
2417	if (ivhd->flags & IVHD_PREFSUP(1L << 6))
2418	DPRINTF(0," PreFSup");
2419	if (ivhd->flags & IVHD_COHERENT(1L << 5))
2420	DPRINTF(0," Coherent");
2421	if (ivhd->flags & IVHD_IOTLB(1L << 4))
2422	DPRINTF(0," Iotlb");
2423	if (ivhd->flags & IVHD_ISOC(1L << 3))
2424	DPRINTF(0," ISoc");
2425	if (ivhd->flags & IVHD_RESPASSPW(1L << 2))
2426	DPRINTF(0," ResPassPW");
2427	if (ivhd->flags & IVHD_PASSPW(1L << 1))
2428	DPRINTF(0," PassPW");
2429	if (ivhd->flags & IVHD_HTTUNEN(1L << 0))
2430	DPRINTF(0, " HtTunEn");
2431	if (ivhd->flags)
2432	DPRINTF(0,"\n");
2433	off = sizeof(*ivhd);
2434	}
2435	while (off < ivhd->length) {
2436	ie = (void *)ivhd + off;
2437	switch (ie->type) {
2438	case IVHD_ALL1:
2439	all_dte = ie->all.data;
2440	DPRINTF(0," ALL %.4x\n", dte);
2441	off += sizeof(ie->all);
2442	break;
2443	case IVHD_SEL2:
2444	dte = ie->sel.data;
2445	DPRINTF(0," SELECT: %s %.4x\n", dmar_bdf(ie->sel.devid), dte);
2446	off += sizeof(ie->sel);
2447	break;
2448	case IVHD_SOR3:
2449	dte = ie->sor.data;
2450	start = ie->sor.devid;
2451	DPRINTF(0," SOR: %s %.4x\n", dmar_bdf(start), dte);
2452	off += sizeof(ie->sor);
2453	break;
2454	case IVHD_EOR4:
2455	DPRINTF(0," EOR: %s\n", dmar_bdf(ie->eor.devid));
2456	off += sizeof(ie->eor);
2457	break;
2458	case IVHD_ALIAS_SEL66:
2459	dte = ie->alias.data;
2460	DPRINTF(0," ALIAS: src=%s: ", dmar_bdf(ie->alias.srcid));
2461	DPRINTF(0," %s %.4x\n", dmar_bdf(ie->alias.devid), dte);
2462	off += sizeof(ie->alias);
2463	break;
2464	case IVHD_ALIAS_SOR67:
2465	dte = ie->alias.data;
2466	DPRINTF(0," ALIAS_SOR: %s %.4x ", dmar_bdf(ie->alias.devid), dte);
2467	DPRINTF(0," src=%s\n", dmar_bdf(ie->alias.srcid));
2468	off += sizeof(ie->alias);
2469	break;
2470	case IVHD_EXT_SEL70:
2471	dte = ie->ext.data;
2472	DPRINTF(0," EXT SEL: %s %.4x %.8x\n", dmar_bdf(ie->ext.devid),
2473	dte, ie->ext.extdata);
2474	off += sizeof(ie->ext);
2475	break;
2476	case IVHD_EXT_SOR71:
2477	dte = ie->ext.data;
2478	DPRINTF(0," EXT SOR: %s %.4x %.8x\n", dmar_bdf(ie->ext.devid),
2479	dte, ie->ext.extdata);
2480	off += sizeof(ie->ext);
2481	break;
2482	case IVHD_SPECIAL72:
2483	DPRINTF(0," SPECIAL\n");
2484	off += sizeof(ie->special);
2485	break;
2486	default:
2487	DPRINTF(0," 2:unknown %x\n", ie->type);
2488	off = ivhd->length;
2489	break;
2490	}
2491	}
2492	}
2493
2494	void
2495	acpiivrs_init(struct acpidmar_softc sc, struct acpi_ivrs ivrs)
2496	{
2497	union acpi_ivrs_entry *ie;
2498	int off;
2499
2500	if (!sc->sc_hwdte) {
2501	sc->sc_hwdte = iommu_alloc_hwdte(sc, HWDTE_SIZE(65536 * sizeof(struct ivhd_dte)), &sc->sc_hwdtep);
2502	if (sc->sc_hwdte == NULL((void *)0))
2503	panic("Can't allocate HWDTE!");
2504	}
2505
2506	domain_map_page = domain_map_page_amd;
2507	DPRINTF(0,"IVRS Version: %d\n", ivrs->hdr.revision);
2508	DPRINTF(0," VA Size: %d\n",
2509	(ivrs->ivinfo >> IVRS_VASIZE_SHIFT) & IVRS_VASIZE_MASK);
2510	DPRINTF(0," PA Size: %d\n",
2511	(ivrs->ivinfo >> IVRS_PASIZE_SHIFT) & IVRS_PASIZE_MASK);
2512
2513	TAILQ_INIT(&sc->sc_drhds)do { (&sc->sc_drhds)->tqh_first = ((void *)0); (& sc->sc_drhds)->tqh_last = &(&sc->sc_drhds)-> tqh_first; } while (0);
2514	TAILQ_INIT(&sc->sc_rmrrs)do { (&sc->sc_rmrrs)->tqh_first = ((void *)0); (& sc->sc_rmrrs)->tqh_last = &(&sc->sc_rmrrs)-> tqh_first; } while (0);
2515	TAILQ_INIT(&sc->sc_atsrs)do { (&sc->sc_atsrs)->tqh_first = ((void *)0); (& sc->sc_atsrs)->tqh_last = &(&sc->sc_atsrs)-> tqh_first; } while (0);
2516
2517	DPRINTF(0,"======== IVRS\n");
2518	off = sizeof(*ivrs);
2519	while (off < ivrs->hdr.length) {
2520	ie = (void *)ivrs + off;
2521	switch (ie->type) {
2522	case IVRS_IVHD0x10:
2523	case IVRS_IVHD_EXT0x11:
2524	acpiivrs_ivhd(sc, &ie->ivhd);
2525	break;
2526	case IVRS_IVMD_ALL0x20:
2527	case IVRS_IVMD_SPECIFIED0x21:
2528	case IVRS_IVMD_RANGE0x22:
2529	DPRINTF(0,"ivmd\n");
2530	break;
2531	default:
2532	DPRINTF(0,"1:unknown: %x\n", ie->type);
2533	break;
2534	}
2535	off += ie->length;
2536	}
2537	DPRINTF(0,"======== End IVRS\n");
2538	}
2539
2540	static int
2541	acpiivhd_activate(struct iommu_softc *iommu, int act)
2542	{
2543	switch (act) {
2544	case DVACT_SUSPEND3:
2545	iommu->flags \|= IOMMU_FLAGS_SUSPEND0x4;
2546	break;
2547	case DVACT_RESUME4:
2548	iommu->flags &= ~IOMMU_FLAGS_SUSPEND0x4;
2549	break;
2550	}
2551	return (0);
2552	}
2553
2554	int
2555	acpidmar_activate(struct device *self, int act)
2556	{
2557	struct acpidmar_softc sc = (struct acpidmar_softc )self;
2558	struct iommu_softc *iommu;
2559
2560	printf("called acpidmar_activate %d %p\n", act, sc);
2561
2562	if (sc == NULL((void *)0)) {
2563	return (0);
2564	}
2565
2566	switch (act) {
2567	case DVACT_RESUME4:
2568	TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu) != ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
2569	printf("iommu%d resume\n", iommu->id);
2570	if (iommu->dte) {
2571	acpiivhd_activate(iommu, act);
2572	continue;
2573	}
2574	iommu_flush_write_buffer(iommu);
2575	iommu_set_rtaddr(iommu, iommu->rtaddr);
2576	iommu_write_4(iommu, DMAR_FEDATA_REG0x3c, iommu->fedata);
2577	iommu_write_4(iommu, DMAR_FEADDR_REG0x40, iommu->feaddr);
2578	iommu_write_4(iommu, DMAR_FEUADDR_REG0x44,
2579	iommu->feaddr >> 32);
2580	if ((iommu->flags & (IOMMU_FLAGS_BAD0x2\|IOMMU_FLAGS_SUSPEND0x4)) ==
2581	IOMMU_FLAGS_SUSPEND0x4) {
2582	printf("enable wakeup translation\n");
2583	iommu_enable_translation(iommu, 1);
2584	}
2585	iommu_showcfg(iommu, -1);
2586	}
2587	break;
2588	case DVACT_SUSPEND3:
2589	TAILQ_FOREACH(iommu, &sc->sc_drhds, link)for((iommu) = ((&sc->sc_drhds)->tqh_first); (iommu) != ((void *)0); (iommu) = ((iommu)->link.tqe_next)) {
2590	printf("iommu%d suspend\n", iommu->id);
2591	if (iommu->flags & IOMMU_FLAGS_BAD0x2)
2592	continue;
2593	if (iommu->dte) {
2594	acpiivhd_activate(iommu, act);
2595	continue;
2596	}
2597	iommu->flags \|= IOMMU_FLAGS_SUSPEND0x4;
2598	iommu_enable_translation(iommu, 0);
2599	iommu_showcfg(iommu, -1);
2600	}
2601	break;
2602	}
2603	return (0);
2604	}
2605
2606	int
2607	acpidmar_match(struct device parent, void match, void *aux)
2608	{
2609	struct acpi_attach_args *aaa = aux;
2610	struct acpi_table_header *hdr;
2611
2612	/* If we do not have a table, it is not us */
2613	if (aaa->aaa_table == NULL((void *)0))
2614	return (0);
2615
2616	/* If it is an DMAR table, we can attach */
2617	hdr = (struct acpi_table_header *)aaa->aaa_table;
2618	if (memcmp(hdr->signature, DMAR_SIG, sizeof(DMAR_SIG) - 1)__builtin_memcmp((hdr->signature), ("DMAR"), (sizeof("DMAR" ) - 1)) == 0)
2619	return (1);
2620	if (memcmp(hdr->signature, IVRS_SIG, sizeof(IVRS_SIG) - 1)__builtin_memcmp((hdr->signature), ("IVRS"), (sizeof("IVRS" ) - 1)) == 0)
2621	return (1);
2622
2623	return (0);
2624	}
2625
2626	void
2627	acpidmar_attach(struct device parent, struct device self, void *aux)
2628	{
2629	struct acpidmar_softc sc = (void )self;
2630	struct acpi_attach_args *aaa = aux;
2631	struct acpi_dmar dmar = (struct acpi_dmar )aaa->aaa_table;
2632	struct acpi_ivrs ivrs = (struct acpi_ivrs )aaa->aaa_table;
2633	struct acpi_table_header *hdr;
2634
2635	hdr = (struct acpi_table_header *)aaa->aaa_table;
2636	sc->sc_memt = aaa->aaa_memt;
2637	sc->sc_dmat = aaa->aaa_dmat;
2638	if (memcmp(hdr->signature, DMAR_SIG, sizeof(DMAR_SIG) - 1)__builtin_memcmp((hdr->signature), ("DMAR"), (sizeof("DMAR" ) - 1)) == 0) {
2639	acpidmar_sc = sc;
2640	acpidmar_init(sc, dmar);
2641	}
2642	if (memcmp(hdr->signature, IVRS_SIG, sizeof(IVRS_SIG) - 1)__builtin_memcmp((hdr->signature), ("IVRS"), (sizeof("IVRS" ) - 1)) == 0) {
2643	acpidmar_sc = sc;
2644	acpiivrs_init(sc, ivrs);
2645	}
2646	}
2647
2648	/* Interrupt shiz */
2649	void acpidmar_msi_hwmask(struct pic *, int);
2650	void acpidmar_msi_hwunmask(struct pic *, int);
2651	void acpidmar_msi_addroute(struct pic , struct cpu_info , int, int, int);
2652	void acpidmar_msi_delroute(struct pic , struct cpu_info , int, int, int);
2653
2654	void
2655	acpidmar_msi_hwmask(struct pic *pic, int pin)
2656	{
2657	struct iommu_pic ip = (void )pic;
2658	struct iommu_softc *iommu = ip->iommu;
2659
2660	printf("msi_hwmask\n");
2661
2662	mtx_enter(&iommu->reg_lock);
2663
2664	iommu_write_4(iommu, DMAR_FECTL_REG0x38, FECTL_IM(1LL << 31));
2665	iommu_read_4(iommu, DMAR_FECTL_REG0x38);
2666
2667	mtx_leave(&iommu->reg_lock);
2668	}
2669
2670	void
2671	acpidmar_msi_hwunmask(struct pic *pic, int pin)
2672	{
2673	struct iommu_pic ip = (void )pic;
2674	struct iommu_softc *iommu = ip->iommu;
2675
2676	printf("msi_hwunmask\n");
2677
2678	mtx_enter(&iommu->reg_lock);
2679
2680	iommu_write_4(iommu, DMAR_FECTL_REG0x38, 0);
2681	iommu_read_4(iommu, DMAR_FECTL_REG0x38);
2682
2683	mtx_leave(&iommu->reg_lock);
2684	}
2685
2686	void
2687	acpidmar_msi_addroute(struct pic pic, struct cpu_info ci, int pin, int vec,
2688	int type)
2689	{
2690	struct iommu_pic ip = (void )pic;
2691	struct iommu_softc *iommu = ip->iommu;
2692
2693	mtx_enter(&iommu->reg_lock);
2694
2695	iommu->fedata = vec;
2696	iommu->feaddr = 0xfee00000L \| (ci->ci_apicid << 12);
2697	iommu_write_4(iommu, DMAR_FEDATA_REG0x3c, vec);
2698	iommu_write_4(iommu, DMAR_FEADDR_REG0x40, iommu->feaddr);
2699	iommu_write_4(iommu, DMAR_FEUADDR_REG0x44, iommu->feaddr >> 32);
2700
2701	mtx_leave(&iommu->reg_lock);
2702	}
2703
2704	void
2705	acpidmar_msi_delroute(struct pic pic, struct cpu_info ci, int pin, int vec,
2706	int type)
2707	{
2708	printf("msi_delroute\n");
2709	}
2710
2711	void *
2712	acpidmar_intr_establish(void ctx, int level, int (func)(void *),
2713	void arg, const char what)
2714	{
2715	struct iommu_softc *iommu = ctx;
2716	struct pic *pic;
2717
2718	pic = &iommu->pic.pic;
2719	iommu->pic.iommu = iommu;
2720
2721	strlcpy(pic->pic_dev.dv_xname, "dmarpic",
2722	sizeof(pic->pic_dev.dv_xname));
2723	pic->pic_type = PIC_MSI3;
2724	pic->pic_hwmask = acpidmar_msi_hwmask;
2725	pic->pic_hwunmask = acpidmar_msi_hwunmask;
2726	pic->pic_addroute = acpidmar_msi_addroute;
2727	pic->pic_delroute = acpidmar_msi_delroute;
2728	pic->pic_edge_stubs = ioapic_edge_stubs;
2729	#ifdef MULTIPROCESSOR1
2730	mtx_init(&pic->pic_mutex, level)do { (void)(((void *)0)); (void)(0); __mtx_init((&pic-> pic_mutex), ((((level)) > 0x0 && ((level)) < 0x9 ) ? 0x9 : ((level)))); } while (0);
2731	#endif
2732
2733	return intr_establish(-1, pic, 0, IST_PULSE1, level, NULL((void *)0), func, arg, what);
2734	}
2735
2736	/* Intel: Handle DMAR Interrupt */
2737	int
2738	acpidmar_intr(void *ctx)
2739	{
2740	struct iommu_softc *iommu = ctx;
2741	struct fault_entry fe;
2742	static struct fault_entry ofe;
2743	int fro, nfr, fri, i;
2744	uint32_t sts;
2745
2746	/splassert(IPL_HIGH);/
2747
2748	if (!(iommu->gcmd & GCMD_TE(1LL << 31))) {
2749	return (1);
2750	}
2751	mtx_enter(&iommu->reg_lock);
2752	sts = iommu_read_4(iommu, DMAR_FECTL_REG0x38);
2753	sts = iommu_read_4(iommu, DMAR_FSTS_REG0x34);
2754
2755	if (!(sts & FSTS_PPF(1LL << 1))) {
2756	mtx_leave(&iommu->reg_lock);
2757	return (1);
2758	}
2759
2760	nfr = cap_nfr(iommu->cap)((uint32_t)(((iommu->cap)>> 40LL) & 0xFF) + 1);
2761	fro = cap_fro(iommu->cap)((uint32_t)(((iommu->cap)>> 24LL) & 0x3FF) * 16);
2762	fri = (sts >> FSTS_FRI_SHIFT8) & FSTS_FRI_MASK0xFF;
2763	for (i = 0; i < nfr; i++) {
2764	fe.hi = iommu_read_8(iommu, fro + (fri*16) + 8);
2765	if (!(fe.hi & FRCD_HI_F(1LL << (127-64))))
2766	break;
2767
2768	fe.lo = iommu_read_8(iommu, fro + (fri*16));
2769	if (ofe.hi != fe.hi \|\| ofe.lo != fe.lo) {
2770	iommu_showfault(iommu, fri, &fe);
2771	ofe.hi = fe.hi;
2772	ofe.lo = fe.lo;
2773	}
2774	fri = (fri + 1) % nfr;
2775	}
2776
2777	iommu_write_4(iommu, DMAR_FSTS_REG0x34, FSTS_PFO(1LL << 0) \| FSTS_PPF(1LL << 1));
2778
2779	mtx_leave(&iommu->reg_lock);
2780
2781	return (1);
2782	}
2783
2784	const char *vtd_faults[] = {
2785	"Software",
2786	"Root Entry Not Present", /* ok (rtaddr + 4096) */
2787	"Context Entry Not Present", /* ok (no CTX_P) */
2788	"Context Entry Invalid", /* ok (tt = 3) */
2789	"Address Beyond MGAW",
2790	"Write", /* ok */
2791	"Read", /* ok */
2792	"Paging Entry Invalid", /* ok */
2793	"Root Table Invalid",
2794	"Context Table Invalid",
2795	"Root Entry Reserved", /* ok (root.lo \|= 0x4) */
2796	"Context Entry Reserved",
2797	"Paging Entry Reserved",
2798	"Context Entry TT",
2799	"Reserved",
2800	};
2801
2802	void iommu_showpte(uint64_t, int, uint64_t);
2803
2804	/* Intel: Show IOMMU page table entry */
2805	void
2806	iommu_showpte(uint64_t ptep, int lvl, uint64_t base)
2807	{
2808	uint64_t nb, pb, i;
2809	struct pte_entry *pte;
2810
2811	pte = (void )PMAP_DIRECT_MAP(ptep)((vaddr_t)(((((511 - 4) (1ULL << 39))) \| 0xffff000000000000 )) + (ptep));
2812	for (i = 0; i < 512; i++) {
2813	if (!(pte[i].val & PTE_P(1L << 0)))
2814	continue;
2815	nb = base + (i << lvl);
2816	pb = pte[i].val & ~VTD_PAGE_MASK0xFFF;
2817	if(lvl == VTD_LEVEL012) {
2818	printf(" %3llx %.16llx = %.16llx %c%c %s\n",
2819	i, nb, pb,
2820	pte[i].val == PTE_R0x00 ? 'r' : ' ',
2821	pte[i].val & PTE_W(1L << 1) ? 'w' : ' ',
2822	(nb == pb) ? " ident" : "");
2823	if (nb == pb)
2824	return;
2825	} else {
2826	iommu_showpte(pb, lvl - VTD_STRIDE_SIZE9, nb);
2827	}
2828	}
2829	}
2830
2831	/* Intel: Show IOMMU configuration */
2832	void
2833	iommu_showcfg(struct iommu_softc *iommu, int sid)
2834	{
2835	int i, j, sts, cmd;
2836	struct context_entry *ctx;
2837	pcitag_t tag;
2838	pcireg_t clc;
2839
2840	cmd = iommu_read_4(iommu, DMAR_GCMD_REG0x18);
2841	sts = iommu_read_4(iommu, DMAR_GSTS_REG0x1c);
2842	printf("iommu%d: flags:%d root pa:%.16llx %s %s %s %.8x %.8x\n",
2843	iommu->id, iommu->flags, iommu_read_8(iommu, DMAR_RTADDR_REG0x20),
2844	sts & GSTS_TES(1LL << 31) ? "enabled" : "disabled",
2845	sts & GSTS_QIES(1LL << 26) ? "qi" : "ccmd",
2846	sts & GSTS_IRES(1LL << 25) ? "ir" : "",
2847	cmd, sts);
2848	for (i = 0; i < 256; i++) {
2849	if (!root_entry_is_valid(&iommu->root[i])) {
2850	continue;
2851	}
2852	for (j = 0; j < 256; j++) {
2853	ctx = iommu->ctx[i] + j;
2854	if (!context_entry_is_valid(ctx)) {
2855	continue;
2856	}
2857	tag = pci_make_tag(NULL((void *)0), i, (j >> 3), j & 0x7);
2858	clc = pci_conf_read(NULL((void *)0), tag, 0x08) >> 8;
2859	printf(" %.2x:%.2x.%x lvl:%d did:%.4x tt:%d ptep:%.16llx flag:%x cc:%.6x\n",
2860	i, (j >> 3), j & 7,
2861	context_address_width(ctx),
2862	context_domain_id(ctx),
2863	context_translation_type(ctx),
2864	context_pte(ctx),
2865	context_user(ctx),
2866	clc);
2867	#if 0
2868	/* dump pagetables */
2869	iommu_showpte(ctx->lo & ~VTD_PAGE_MASK0xFFF, iommu->agaw -
2870	VTD_STRIDE_SIZE9, 0);
2871	#endif
2872	}
2873	}
2874	}
2875
2876	/* Intel: Show IOMMU fault */
2877	void
2878	iommu_showfault(struct iommu_softc iommu, int fri, struct fault_entry fe)
2879	{
2880	int bus, dev, fun, type, fr, df;
2881	bios_memmap_t *im;
2882	const char *mapped;
2883
2884	if (!(fe->hi & FRCD_HI_F(1LL << (127-64))))
2885	return;
2886	type = (fe->hi & FRCD_HI_T(1LL << (126-64))) ? 'r' : 'w';
2887	fr = (fe->hi >> FRCD_HI_FR_SHIFT(96-64)) & FRCD_HI_FR_MASK0xFF;
2888	bus = (fe->hi >> FRCD_HI_BUS_SHIFT8) & FRCD_HI_BUS_MASK0xFF;
2889	dev = (fe->hi >> FRCD_HI_DEV_SHIFT3) & FRCD_HI_DEV_MASK0x1F;
2890	fun = (fe->hi >> FRCD_HI_FUN_SHIFT0) & FRCD_HI_FUN_MASK0x7;
2891	df = (fe->hi >> FRCD_HI_FUN_SHIFT0) & 0xFF;
2892	iommu_showcfg(iommu, mksid(bus,dev,fun));
2893	if (!iommu->ctx[bus]) {
2894	/* Bus is not initialized */
2895	mapped = "nobus";
2896	} else if (!context_entry_is_valid(&iommu->ctx[bus][df])) {
2897	/* DevFn not initialized */
2898	mapped = "nodevfn";
2899	} else if (context_user(&iommu->ctx[bus][df]) != 0xA) {
2900	/* no bus_space_map */
2901	mapped = "nomap";
2902	} else {
2903	/* bus_space_map */
2904	mapped = "mapped";
2905	}
2906	printf("fri%d: dmar: %.2x:%.2x.%x %s error at %llx fr:%d [%s] iommu:%d [%s]\n",
2907	fri, bus, dev, fun,
2908	type == 'r' ? "read" : "write",
2909	fe->lo,
2910	fr, fr <= 13 ? vtd_faults[fr] : "unknown",
2911	iommu->id,
2912	mapped);
2913	for (im = bios_memmap; im->type != BIOS_MAP_END0x00; im++) {
2914	if ((im->type == BIOS_MAP_RES0x02) &&
2915	(im->addr <= fe->lo) &&
2916	(fe->lo <= im->addr+im->size)) {
2917	printf("mem in e820.reserved\n");
2918	}
2919	}
2920	#ifdef DDB1
2921	if (acpidmar_ddb)
2922	db_enter();
2923	#endif
2924	}
2925