Bug Summary

File:dev/pci/drm/i915/gt/intel_ggtt.c
Warning:line 962, column 18
Value stored to 'pdev' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name intel_ggtt.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/gt/intel_ggtt.c
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5
6#include <asm/set_memory.h>
7#include <asm/smp.h>
8#include <linux/types.h>
9#include <linux/stop_machine.h>
10
11#include <drm/i915_drm.h>
12#include <drm/intel-gtt.h>
13
14#include "gem/i915_gem_lmem.h"
15
16#include "intel_ggtt_gmch.h"
17#include "intel_gt.h"
18#include "intel_gt_regs.h"
19#include "intel_pci_config.h"
20#include "i915_drv.h"
21#include "i915_pci.h"
22#include "i915_scatterlist.h"
23#include "i915_utils.h"
24#include "i915_vgpu.h"
25
26#include "intel_gtt.h"
27#include "gen8_ppgtt.h"
28
29#include <dev/pci/pcivar.h>
30#include <dev/pci/agpvar.h>
31
32static inline bool_Bool suspend_retains_ptes(struct i915_address_space *vm)
33{
34 return GRAPHICS_VER(vm->i915)((&(vm->i915)->__runtime)->graphics.ip.ver) >= 8 &&
35 !HAS_LMEM(vm->i915)((&(vm->i915)->__runtime)->memory_regions & (
(1UL << (INTEL_REGION_LMEM_0)))) &&
36 vm->is_ggtt;
37}
38
39static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
40 unsigned long color,
41 u64 *start,
42 u64 *end)
43{
44 if (i915_node_color_differs(node, color))
45 *start += I915_GTT_PAGE_SIZE(1ULL << (12));
46
47 /*
48 * Also leave a space between the unallocated reserved node after the
49 * GTT and any objects within the GTT, i.e. we use the color adjustment
50 * to insert a guard page to prevent prefetches crossing over the
51 * GTT boundary.
52 */
53 node = list_next_entry(node, node_list)({ const __typeof( ((typeof(*(node)) *)0)->node_list ) *__mptr
= (((node)->node_list.next)); (typeof(*(node)) *)( (char *
)__mptr - __builtin_offsetof(typeof(*(node)), node_list) );});
54 if (node->color != color)
55 *end -= I915_GTT_PAGE_SIZE(1ULL << (12));
56}
57
58static int ggtt_init_hw(struct i915_ggtt *ggtt)
59{
60 struct drm_i915_privateinteldrm_softc *i915 = ggtt->vm.i915;
61 int i;
62
63 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT0);
64
65 ggtt->vm.is_ggtt = true1;
66
67 /* Only VLV supports read-only GGTT mappings */
68 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915)IS_PLATFORM(i915, INTEL_VALLEYVIEW);
69
70 if (!HAS_LLC(i915)((&(i915)->__info)->has_llc) && !HAS_PPGTT(i915)(((&(i915)->__runtime)->ppgtt_type) != INTEL_PPGTT_NONE
))
71 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
72
73 if (ggtt->mappable_end) {
74#ifdef __linux__
75 if (!io_mapping_init_wc(&ggtt->iomap,
76 ggtt->gmadr.start,
77 ggtt->mappable_end)) {
78 ggtt->vm.cleanup(&ggtt->vm);
79 return -EIO5;
80 }
81
82 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
83 ggtt->mappable_end);
84#else
85 /* XXX would be a lot nicer to get agp info before now */
86 uvm_page_physload(atop(ggtt->gmadr.start)((ggtt->gmadr.start) >> 12),
87 atop(ggtt->gmadr.start + ggtt->mappable_end)((ggtt->gmadr.start + ggtt->mappable_end) >> 12),
88 atop(ggtt->gmadr.start)((ggtt->gmadr.start) >> 12),
89 atop(ggtt->gmadr.start + ggtt->mappable_end)((ggtt->gmadr.start + ggtt->mappable_end) >> 12),
90 PHYSLOAD_DEVICE0x01);
91 /* array of vm pages that physload introduced. */
92 i915->pgs = PHYS_TO_VM_PAGE(ggtt->gmadr.start);
93 KASSERT(i915->pgs != NULL)((i915->pgs != ((void *)0)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/pci/drm/i915/gt/intel_ggtt.c", 93, "i915->pgs != NULL"
));
94 /*
95 * XXX mark all pages write combining so user mmaps get the
96 * right bits. We really need a proper MI api for doing this,
97 * but for now this allows us to use PAT where available.
98 */
99 for (i = 0; i < atop(ggtt->mappable_end)((ggtt->mappable_end) >> 12); i++)
100 atomic_setbits_intx86_atomic_setbits_u32(&(i915->pgs[i].pg_flags),
101 PG_PMAP_WC0x04000000);
102 if (agp_init_map(i915->bst, ggtt->gmadr.start,
103 ggtt->mappable_end,
104 BUS_SPACE_MAP_LINEAR0x0002 | BUS_SPACE_MAP_PREFETCHABLE0x0008,
105 &i915->agph))
106 panic("can't map aperture");
107#endif
108 }
109
110 intel_ggtt_init_fences(ggtt);
111
112 return 0;
113}
114
115/**
116 * i915_ggtt_init_hw - Initialize GGTT hardware
117 * @i915: i915 device
118 */
119int i915_ggtt_init_hw(struct drm_i915_privateinteldrm_softc *i915)
120{
121 int ret;
122
123 /*
124 * Note that we use page colouring to enforce a guard page at the
125 * end of the address space. This is required as the CS may prefetch
126 * beyond the end of the batch buffer, across the page boundary,
127 * and beyond the end of the GTT if we do not provide a guard.
128 */
129 ret = ggtt_init_hw(to_gt(i915)->ggtt);
130 if (ret)
131 return ret;
132
133 return 0;
134}
135
136/*
137 * Return the value of the last GGTT pte cast to an u64, if
138 * the system is supposed to retain ptes across resume. 0 otherwise.
139 */
140static u64 read_last_pte(struct i915_address_space *vm)
141{
142 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
143 gen8_pte_t __iomem *ptep;
144
145 if (!suspend_retains_ptes(vm))
146 return 0;
147
148 GEM_BUG_ON(GRAPHICS_VER(vm->i915) < 8)((void)0);
149 ptep = (typeof(ptep))ggtt->gsm + (ggtt_total_entries(ggtt)((ggtt)->vm.total >> 12) - 1);
150 return readq(ptep)ioread64(ptep);
151}
152
153/**
154 * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM
155 * @vm: The VM to suspend the mappings for
156 *
157 * Suspend the memory mappings for all objects mapped to HW via the GGTT or a
158 * DPT page table.
159 */
160void i915_ggtt_suspend_vm(struct i915_address_space *vm)
161{
162 struct i915_vma *vma, *vn;
163 int save_skip_rewrite;
164
165 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt)({ int __ret = !!((!vm->is_ggtt && !vm->is_dpt)
); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
vm->i915->drm))->dev), "", "drm_WARN_ON(" "!vm->is_ggtt && !vm->is_dpt"
")"); __builtin_expect(!!(__ret), 0); });
166
167retry:
168 i915_gem_drain_freed_objects(vm->i915);
169
170 mutex_lock(&vm->mutex)rw_enter_write(&vm->mutex);
171
172 /*
173 * Skip rewriting PTE on VMA unbind.
174 * FIXME: Use an argument to i915_vma_unbind() instead?
175 */
176 save_skip_rewrite = vm->skip_pte_rewrite;
177 vm->skip_pte_rewrite = true1;
178
179 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link
) *__mptr = ((&vm->bound_list)->next); (__typeof(*
vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), vm_link
) );}), vn = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link
) *__mptr = (vma->vm_link.next); (__typeof(*vma) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*vma), vm_link) );});
&vma->vm_link != (&vm->bound_list); vma = vn, vn
= ({ const __typeof( ((__typeof(*vn) *)0)->vm_link ) *__mptr
= (vn->vm_link.next); (__typeof(*vn) *)( (char *)__mptr -
__builtin_offsetof(__typeof(*vn), vm_link) );})) {
180 struct drm_i915_gem_object *obj = vma->obj;
181
182 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0);
183
184 if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10)))))
185 continue;
186
187 /* unlikely to race when GPU is idle, so no worry about slowpath.. */
188 if (WARN_ON(!i915_gem_object_trylock(obj, NULL))({ int __ret = !!(!i915_gem_object_trylock(obj, ((void *)0)))
; if (__ret) printf("WARNING %s failed at %s:%d\n", "!i915_gem_object_trylock(obj, ((void *)0))"
, "/usr/src/sys/dev/pci/drm/i915/gt/intel_ggtt.c", 188); __builtin_expect
(!!(__ret), 0); })) {
189 /*
190 * No dead objects should appear here, GPU should be
191 * completely idle, and userspace suspended
192 */
193 i915_gem_object_get(obj);
194
195 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
196
197 i915_gem_object_lock(obj, NULL((void *)0));
198 GEM_WARN_ON(i915_vma_unbind(vma))({ __builtin_expect(!!(!!(i915_vma_unbind(vma))), 0); });
199 i915_gem_object_unlock(obj);
200 i915_gem_object_put(obj);
201
202 vm->skip_pte_rewrite = save_skip_rewrite;
203 goto retry;
204 }
205
206 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10))))) {
207 i915_vma_wait_for_bind(vma);
208
209 __i915_vma_evict(vma, false0);
210 drm_mm_remove_node(&vma->node);
211 }
212
213 i915_gem_object_unlock(obj);
214 }
215
216 if (!suspend_retains_ptes(vm))
217 vm->clear_range(vm, 0, vm->total);
218 else
219 i915_vm_to_ggtt(vm)->probed_pte = read_last_pte(vm);
220
221 vm->skip_pte_rewrite = save_skip_rewrite;
222
223 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
224}
225
226void i915_ggtt_suspend(struct i915_ggtt *ggtt)
227{
228 i915_ggtt_suspend_vm(&ggtt->vm);
229 ggtt->invalidate(ggtt);
230
231 intel_gt_check_and_clear_faults(ggtt->vm.gt);
232}
233
234void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
235{
236 struct intel_uncore *uncore = ggtt->vm.gt->uncore;
237
238 spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock);
239 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x101008
) }), (1 << 0));
240 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = (0x101008
) }));
241 spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock);
242}
243
244static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
245{
246 struct intel_uncore *uncore = ggtt->vm.gt->uncore;
247
248 /*
249 * Note that as an uncached mmio write, this will flush the
250 * WCB of the writes into the GGTT before it triggers the invalidate.
251 */
252 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x101008
) }), (1 << 0));
253}
254
255static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
256{
257 struct intel_uncore *uncore = ggtt->vm.gt->uncore;
258 struct drm_i915_privateinteldrm_softc *i915 = ggtt->vm.i915;
259
260 gen8_ggtt_invalidate(ggtt);
261
262 if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 12)
263 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR,__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0xcee8
) }), (1 << 0))
264 GEN12_GUC_TLB_INV_CR_INVALIDATE)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0xcee8
) }), (1 << 0));
265 else
266 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x4274
) }), (1<<0));
267}
268
269u64 gen8_ggtt_pte_encode(dma_addr_t addr,
270 enum i915_cache_level level,
271 u32 flags)
272{
273 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT(1ULL << (0));
274
275 if (flags & PTE_LM(1UL << (1)))
276 pte |= GEN12_GGTT_PTE_LM(1ULL << (1));
277
278 return pte;
279}
280
281static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
282{
283 writeq(pte, addr)iowrite64(pte, addr);
284}
285
286static void gen8_ggtt_insert_page(struct i915_address_space *vm,
287 dma_addr_t addr,
288 u64 offset,
289 enum i915_cache_level level,
290 u32 flags)
291{
292 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
293 gen8_pte_t __iomem *pte =
294 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE(1ULL << (12));
295
296 gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, flags));
297
298 ggtt->invalidate(ggtt);
299}
300
301static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
302 struct i915_vma_resource *vma_res,
303 enum i915_cache_level level,
304 u32 flags)
305{
306 const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, flags);
307 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
308 gen8_pte_t __iomem *gte;
309 gen8_pte_t __iomem *end;
310 struct sgt_iter iter;
311 dma_addr_t addr;
312
313 /*
314 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
315 * not to allow the user to override access to a read only page.
316 */
317
318 gte = (gen8_pte_t __iomem *)ggtt->gsm;
319 gte += vma_res->start / I915_GTT_PAGE_SIZE(1ULL << (12));
320 end = gte + vma_res->node_size / I915_GTT_PAGE_SIZE(1ULL << (12));
321
322 for_each_sgt_daddr(addr, iter, vma_res->bi.pages)for ((iter) = __sgt_iter((vma_res->bi.pages)->sgl, 1); (
(addr) = (iter).dma + (iter).curr), (iter).sgp; (((iter).curr
+= ((1ULL << (12)))) >= (iter).max) ? (iter) = __sgt_iter
(__sg_next((iter).sgp), 1), 0 : 0)
323 gen8_set_pte(gte++, pte_encode | addr);
324 GEM_BUG_ON(gte > end)((void)0);
325
326 /* Fill the allocated but "unused" space beyond the end of the buffer */
327 while (gte < end)
328 gen8_set_pte(gte++, vm->scratch[0]->encode);
329
330 /*
331 * We want to flush the TLBs only after we're certain all the PTE
332 * updates have finished.
333 */
334 ggtt->invalidate(ggtt);
335}
336
337static void gen6_ggtt_insert_page(struct i915_address_space *vm,
338 dma_addr_t addr,
339 u64 offset,
340 enum i915_cache_level level,
341 u32 flags)
342{
343 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
344 gen6_pte_t __iomem *pte =
345 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE(1ULL << (12));
346
347 iowrite32(vm->pte_encode(addr, level, flags), pte);
348
349 ggtt->invalidate(ggtt);
350}
351
352/*
353 * Binds an object into the global gtt with the specified cache level.
354 * The object will be accessible to the GPU via commands whose operands
355 * reference offsets within the global GTT as well as accessible by the GPU
356 * through the GMADR mapped BAR (i915->mm.gtt->gtt).
357 */
358static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
359 struct i915_vma_resource *vma_res,
360 enum i915_cache_level level,
361 u32 flags)
362{
363 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
364 gen6_pte_t __iomem *gte;
365 gen6_pte_t __iomem *end;
366 struct sgt_iter iter;
367 dma_addr_t addr;
368
369 gte = (gen6_pte_t __iomem *)ggtt->gsm;
370 gte += vma_res->start / I915_GTT_PAGE_SIZE(1ULL << (12));
371 end = gte + vma_res->node_size / I915_GTT_PAGE_SIZE(1ULL << (12));
372
373 for_each_sgt_daddr(addr, iter, vma_res->bi.pages)for ((iter) = __sgt_iter((vma_res->bi.pages)->sgl, 1); (
(addr) = (iter).dma + (iter).curr), (iter).sgp; (((iter).curr
+= ((1ULL << (12)))) >= (iter).max) ? (iter) = __sgt_iter
(__sg_next((iter).sgp), 1), 0 : 0)
374 iowrite32(vm->pte_encode(addr, level, flags), gte++);
375 GEM_BUG_ON(gte > end)((void)0);
376
377 /* Fill the allocated but "unused" space beyond the end of the buffer */
378 while (gte < end)
379 iowrite32(vm->scratch[0]->encode, gte++);
380
381 /*
382 * We want to flush the TLBs only after we're certain all the PTE
383 * updates have finished.
384 */
385 ggtt->invalidate(ggtt);
386}
387
388static void nop_clear_range(struct i915_address_space *vm,
389 u64 start, u64 length)
390{
391}
392
393static void gen8_ggtt_clear_range(struct i915_address_space *vm,
394 u64 start, u64 length)
395{
396 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
397 unsigned int first_entry = start / I915_GTT_PAGE_SIZE(1ULL << (12));
398 unsigned int num_entries = length / I915_GTT_PAGE_SIZE(1ULL << (12));
399 const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
400 gen8_pte_t __iomem *gtt_base =
401 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
402 const int max_entries = ggtt_total_entries(ggtt)((ggtt)->vm.total >> 12) - first_entry;
403 int i;
404
405 if (WARN(num_entries > max_entries,({ int __ret = !!(num_entries > max_entries); if (__ret) printf
("First entry = %d; Num entries = %d (max=%d)\n", first_entry
, num_entries, max_entries); __builtin_expect(!!(__ret), 0); }
)
406 "First entry = %d; Num entries = %d (max=%d)\n",({ int __ret = !!(num_entries > max_entries); if (__ret) printf
("First entry = %d; Num entries = %d (max=%d)\n", first_entry
, num_entries, max_entries); __builtin_expect(!!(__ret), 0); }
)
407 first_entry, num_entries, max_entries)({ int __ret = !!(num_entries > max_entries); if (__ret) printf
("First entry = %d; Num entries = %d (max=%d)\n", first_entry
, num_entries, max_entries); __builtin_expect(!!(__ret), 0); }
))
408 num_entries = max_entries;
409
410 for (i = 0; i < num_entries; i++)
411 gen8_set_pte(&gtt_base[i], scratch_pte);
412}
413
414static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
415{
416 /*
417 * Make sure the internal GAM fifo has been cleared of all GTT
418 * writes before exiting stop_machine(). This guarantees that
419 * any aperture accesses waiting to start in another process
420 * cannot back up behind the GTT writes causing a hang.
421 * The register can be any arbitrary GAM register.
422 */
423 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6)((void)__raw_uncore_read32(vm->gt->uncore, ((const i915_reg_t
){ .reg = (0x101008) })));
424}
425
426struct insert_page {
427 struct i915_address_space *vm;
428 dma_addr_t addr;
429 u64 offset;
430 enum i915_cache_level level;
431};
432
433static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
434{
435 struct insert_page *arg = _arg;
436
437 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
438 bxt_vtd_ggtt_wa(arg->vm);
439
440 return 0;
441}
442
443static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
444 dma_addr_t addr,
445 u64 offset,
446 enum i915_cache_level level,
447 u32 unused)
448{
449 struct insert_page arg = { vm, addr, offset, level };
450
451 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL((void *)0));
452}
453
454struct insert_entries {
455 struct i915_address_space *vm;
456 struct i915_vma_resource *vma_res;
457 enum i915_cache_level level;
458 u32 flags;
459};
460
461static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
462{
463 struct insert_entries *arg = _arg;
464
465 gen8_ggtt_insert_entries(arg->vm, arg->vma_res, arg->level, arg->flags);
466 bxt_vtd_ggtt_wa(arg->vm);
467
468 return 0;
469}
470
471static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
472 struct i915_vma_resource *vma_res,
473 enum i915_cache_level level,
474 u32 flags)
475{
476 struct insert_entries arg = { vm, vma_res, level, flags };
477
478 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL((void *)0));
479}
480
481static void gen6_ggtt_clear_range(struct i915_address_space *vm,
482 u64 start, u64 length)
483{
484 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
485 unsigned int first_entry = start / I915_GTT_PAGE_SIZE(1ULL << (12));
486 unsigned int num_entries = length / I915_GTT_PAGE_SIZE(1ULL << (12));
487 gen6_pte_t scratch_pte, __iomem *gtt_base =
488 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
489 const int max_entries = ggtt_total_entries(ggtt)((ggtt)->vm.total >> 12) - first_entry;
490 int i;
491
492 if (WARN(num_entries > max_entries,({ int __ret = !!(num_entries > max_entries); if (__ret) printf
("First entry = %d; Num entries = %d (max=%d)\n", first_entry
, num_entries, max_entries); __builtin_expect(!!(__ret), 0); }
)
493 "First entry = %d; Num entries = %d (max=%d)\n",({ int __ret = !!(num_entries > max_entries); if (__ret) printf
("First entry = %d; Num entries = %d (max=%d)\n", first_entry
, num_entries, max_entries); __builtin_expect(!!(__ret), 0); }
)
494 first_entry, num_entries, max_entries)({ int __ret = !!(num_entries > max_entries); if (__ret) printf
("First entry = %d; Num entries = %d (max=%d)\n", first_entry
, num_entries, max_entries); __builtin_expect(!!(__ret), 0); }
))
495 num_entries = max_entries;
496
497 scratch_pte = vm->scratch[0]->encode;
498 for (i = 0; i < num_entries; i++)
499 iowrite32(scratch_pte, &gtt_base[i]);
500}
501
502void intel_ggtt_bind_vma(struct i915_address_space *vm,
503 struct i915_vm_pt_stash *stash,
504 struct i915_vma_resource *vma_res,
505 enum i915_cache_level cache_level,
506 u32 flags)
507{
508 u32 pte_flags;
509
510 if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11))))))
511 return;
512
513 vma_res->bound_flags |= flags;
514
515 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
516 pte_flags = 0;
517 if (vma_res->bi.readonly)
518 pte_flags |= PTE_READ_ONLY(1UL << (0));
519 if (vma_res->bi.lmem)
520 pte_flags |= PTE_LM(1UL << (1));
521
522 vm->insert_entries(vm, vma_res, cache_level, pte_flags);
523 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE(1ULL << (12));
524}
525
526void intel_ggtt_unbind_vma(struct i915_address_space *vm,
527 struct i915_vma_resource *vma_res)
528{
529 vm->clear_range(vm, vma_res->start, vma_res->vma_size);
530}
531
532/*
533 * Reserve the top of the GuC address space for firmware images. Addresses
534 * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC,
535 * which makes for a suitable range to hold GuC/HuC firmware images if the
536 * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT
537 * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk
538 * of the same size anyway, which is far more than needed, to keep the logic
539 * in uc_fw_ggtt_offset() simple.
540 */
541#define GUC_TOP_RESERVE_SIZE((4ULL << 30) - 0xFEE00000) (SZ_4G(4ULL << 30) - GUC_GGTT_TOP0xFEE00000)
542
543static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
544{
545 u64 offset;
546 int ret;
547
548 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
549 return 0;
550
551 GEM_BUG_ON(ggtt->vm.total <= GUC_TOP_RESERVE_SIZE)((void)0);
552 offset = ggtt->vm.total - GUC_TOP_RESERVE_SIZE((4ULL << 30) - 0xFEE00000);
553
554 ret = i915_gem_gtt_reserve(&ggtt->vm, NULL((void *)0), &ggtt->uc_fw,
555 GUC_TOP_RESERVE_SIZE((4ULL << 30) - 0xFEE00000), offset,
556 I915_COLOR_UNEVICTABLE(-1), PIN_NOEVICT(1ULL << (0)));
557 if (ret)
558 drm_dbg(&ggtt->vm.i915->drm,__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "Failed to reserve top of GGTT for GuC\n")
559 "Failed to reserve top of GGTT for GuC\n")__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "Failed to reserve top of GGTT for GuC\n");
560
561 return ret;
562}
563
564static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
565{
566 if (drm_mm_node_allocated(&ggtt->uc_fw))
567 drm_mm_remove_node(&ggtt->uc_fw);
568}
569
570static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
571{
572 ggtt_release_guc_top(ggtt);
573 if (drm_mm_node_allocated(&ggtt->error_capture))
574 drm_mm_remove_node(&ggtt->error_capture);
575 mutex_destroy(&ggtt->error_mutex);
576}
577
578static int init_ggtt(struct i915_ggtt *ggtt)
579{
580 /*
581 * Let GEM Manage all of the aperture.
582 *
583 * However, leave one page at the end still bound to the scratch page.
584 * There are a number of places where the hardware apparently prefetches
585 * past the end of the object, and we've seen multiple hangs with the
586 * GPU head pointer stuck in a batchbuffer bound at the last page of the
587 * aperture. One page should be enough to keep any prefetching inside
588 * of the aperture.
589 */
590 unsigned long hole_start, hole_end;
591 struct drm_mm_node *entry;
592 int ret;
593
594 ggtt->pte_lost = true1;
595
596 /*
597 * GuC requires all resources that we're sharing with it to be placed in
598 * non-WOPCM memory. If GuC is not present or not in use we still need a
599 * small bias as ring wraparound at offset 0 sometimes hangs. No idea
600 * why.
601 */
602 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,({ u32 __max_a = ((1ULL << (12))); u32 __max_b = (intel_wopcm_guc_size
(&ggtt->vm.i915->wopcm)); __max_a > __max_b ? __max_a
: __max_b; })
603 intel_wopcm_guc_size(&ggtt->vm.i915->wopcm))({ u32 __max_a = ((1ULL << (12))); u32 __max_b = (intel_wopcm_guc_size
(&ggtt->vm.i915->wopcm)); __max_a > __max_b ? __max_a
: __max_b; });
604
605 ret = intel_vgt_balloon(ggtt);
606 if (ret)
607 return ret;
608
609 rw_init(&ggtt->error_mutex, "ggtter")_rw_init_flags(&ggtt->error_mutex, "ggtter", 0, ((void
*)0));
610 if (ggtt->mappable_end) {
611 /*
612 * Reserve a mappable slot for our lockless error capture.
613 *
614 * We strongly prefer taking address 0x0 in order to protect
615 * other critical buffers against accidental overwrites,
616 * as writing to address 0 is a very common mistake.
617 *
618 * Since 0 may already be in use by the system (e.g. the BIOS
619 * framebuffer), we let the reservation fail quietly and hope
620 * 0 remains reserved always.
621 *
622 * If we fail to reserve 0, and then fail to find any space
623 * for an error-capture, remain silent. We can afford not
624 * to reserve an error_capture node as we have fallback
625 * paths, and we trust that 0 will remain reserved. However,
626 * the only likely reason for failure to insert is a driver
627 * bug, which we expect to cause other failures...
628 */
629 ggtt->error_capture.size = I915_GTT_PAGE_SIZE(1ULL << (12));
630 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE(-1);
631 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
632 drm_mm_insert_node_in_range(&ggtt->vm.mm,
633 &ggtt->error_capture,
634 ggtt->error_capture.size, 0,
635 ggtt->error_capture.color,
636 0, ggtt->mappable_end,
637 DRM_MM_INSERT_LOW);
638 }
639 if (drm_mm_node_allocated(&ggtt->error_capture))
640 drm_dbg(&ggtt->vm.i915->drm,__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "Reserved GGTT:[%llx, %llx] for use by error capture\n", ggtt
->error_capture.start, ggtt->error_capture.start + ggtt
->error_capture.size)
641 "Reserved GGTT:[%llx, %llx] for use by error capture\n",__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "Reserved GGTT:[%llx, %llx] for use by error capture\n", ggtt
->error_capture.start, ggtt->error_capture.start + ggtt
->error_capture.size)
642 ggtt->error_capture.start,__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "Reserved GGTT:[%llx, %llx] for use by error capture\n", ggtt
->error_capture.start, ggtt->error_capture.start + ggtt
->error_capture.size)
643 ggtt->error_capture.start + ggtt->error_capture.size)__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "Reserved GGTT:[%llx, %llx] for use by error capture\n", ggtt
->error_capture.start, ggtt->error_capture.start + ggtt
->error_capture.size);
644
645 /*
646 * The upper portion of the GuC address space has a sizeable hole
647 * (several MB) that is inaccessible by GuC. Reserve this range within
648 * GGTT as it can comfortably hold GuC/HuC firmware images.
649 */
650 ret = ggtt_reserve_guc_top(ggtt);
651 if (ret)
652 goto err;
653
654 /* Clear any non-preallocated blocks */
655 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end)for (entry = ({ const __typeof( ((typeof(*entry) *)0)->hole_stack
) *__mptr = ((&(&ggtt->vm.mm)->hole_stack)->
next); (typeof(*entry) *)( (char *)__mptr - __builtin_offsetof
(typeof(*entry), hole_stack) );}); &entry->hole_stack !=
&(&ggtt->vm.mm)->hole_stack ? hole_start = drm_mm_hole_node_start
(entry), hole_end = hole_start + entry->hole_size, 1 : 0; entry
= ({ const __typeof( ((typeof(*(entry)) *)0)->hole_stack )
*__mptr = (((entry)->hole_stack.next)); (typeof(*(entry))
*)( (char *)__mptr - __builtin_offsetof(typeof(*(entry)), hole_stack
) );})) {
656 drm_dbg(&ggtt->vm.i915->drm,__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "clearing unused GTT space: [%lx, %lx]\n", hole_start, hole_end
)
657 "clearing unused GTT space: [%lx, %lx]\n",__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "clearing unused GTT space: [%lx, %lx]\n", hole_start, hole_end
)
658 hole_start, hole_end)__drm_dev_dbg(((void *)0), (&ggtt->vm.i915->drm) ? (
&ggtt->vm.i915->drm)->dev : ((void *)0), DRM_UT_DRIVER
, "clearing unused GTT space: [%lx, %lx]\n", hole_start, hole_end
);
659 ggtt->vm.clear_range(&ggtt->vm, hole_start,
660 hole_end - hole_start);
661 }
662
663 /* And finally clear the reserved guard page */
664 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE(1 << 12), PAGE_SIZE(1 << 12));
665
666 return 0;
667
668err:
669 cleanup_init_ggtt(ggtt);
670 return ret;
671}
672
673static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
674 struct i915_vm_pt_stash *stash,
675 struct i915_vma_resource *vma_res,
676 enum i915_cache_level cache_level,
677 u32 flags)
678{
679 u32 pte_flags;
680
681 /* Currently applicable only to VLV */
682 pte_flags = 0;
683 if (vma_res->bi.readonly)
684 pte_flags |= PTE_READ_ONLY(1UL << (0));
685
686 if (flags & I915_VMA_LOCAL_BIND((int)(1UL << (11))))
687 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
688 stash, vma_res, cache_level, flags);
689
690 if (flags & I915_VMA_GLOBAL_BIND((int)(1UL << (10))))
691 vm->insert_entries(vm, vma_res, cache_level, pte_flags);
692
693 vma_res->bound_flags |= flags;
694}
695
696static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
697 struct i915_vma_resource *vma_res)
698{
699 if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND((int)(1UL << (10))))
700 vm->clear_range(vm, vma_res->start, vma_res->vma_size);
701
702 if (vma_res->bound_flags & I915_VMA_LOCAL_BIND((int)(1UL << (11))))
703 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);
704}
705
706static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
707{
708 struct i915_vm_pt_stash stash = {};
709 struct i915_ppgtt *ppgtt;
710 int err;
711
712 ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);
713 if (IS_ERR(ppgtt))
714 return PTR_ERR(ppgtt);
715
716 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)({ __builtin_expect(!!(!!(ppgtt->vm.total < ggtt->vm
.total)), 0); })) {
717 err = -ENODEV19;
718 goto err_ppgtt;
719 }
720
721 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
722 if (err)
723 goto err_ppgtt;
724
725 i915_gem_object_lock(ppgtt->vm.scratch[0], NULL((void *)0));
726 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
727 i915_gem_object_unlock(ppgtt->vm.scratch[0]);
728 if (err)
729 goto err_stash;
730
731 /*
732 * Note we only pre-allocate as far as the end of the global
733 * GTT. On 48b / 4-level page-tables, the difference is very,
734 * very significant! We have to preallocate as GVT/vgpu does
735 * not like the page directory disappearing.
736 */
737 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
738
739 ggtt->alias = ppgtt;
740 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
741
742 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma)((void)0);
743 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
744
745 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma)((void)0);
746 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
747
748 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
749 return 0;
750
751err_stash:
752 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
753err_ppgtt:
754 i915_vm_put(&ppgtt->vm);
755 return err;
756}
757
758static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
759{
760 struct i915_ppgtt *ppgtt;
761
762 ppgtt = fetch_and_zero(&ggtt->alias)({ typeof(*&ggtt->alias) __T = *(&ggtt->alias);
*(&ggtt->alias) = (typeof(*&ggtt->alias))0; __T
; });
763 if (!ppgtt)
764 return;
765
766 i915_vm_put(&ppgtt->vm);
767
768 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;
769 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
770}
771
772int i915_init_ggtt(struct drm_i915_privateinteldrm_softc *i915)
773{
774 int ret;
775
776 ret = init_ggtt(to_gt(i915)->ggtt);
777 if (ret)
778 return ret;
779
780 if (INTEL_PPGTT(i915)((&(i915)->__runtime)->ppgtt_type) == INTEL_PPGTT_ALIASING) {
781 ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);
782 if (ret)
783 cleanup_init_ggtt(to_gt(i915)->ggtt);
784 }
785
786 return 0;
787}
788
789static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
790{
791 struct i915_vma *vma, *vn;
792
793 flush_workqueue(ggtt->vm.i915->wq);
794 i915_gem_drain_freed_objects(ggtt->vm.i915);
795
796 mutex_lock(&ggtt->vm.mutex)rw_enter_write(&ggtt->vm.mutex);
797
798 ggtt->vm.skip_pte_rewrite = true1;
799
800 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link
) *__mptr = ((&ggtt->vm.bound_list)->next); (__typeof
(*vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma)
, vm_link) );}), vn = ({ const __typeof( ((__typeof(*vma) *)0
)->vm_link ) *__mptr = (vma->vm_link.next); (__typeof(*
vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), vm_link
) );}); &vma->vm_link != (&ggtt->vm.bound_list)
; vma = vn, vn = ({ const __typeof( ((__typeof(*vn) *)0)->
vm_link ) *__mptr = (vn->vm_link.next); (__typeof(*vn) *)(
(char *)__mptr - __builtin_offsetof(__typeof(*vn), vm_link) )
;})) {
801 struct drm_i915_gem_object *obj = vma->obj;
802 bool_Bool trylock;
803
804 trylock = i915_gem_object_trylock(obj, NULL((void *)0));
805 WARN_ON(!trylock)({ int __ret = !!(!trylock); if (__ret) printf("WARNING %s failed at %s:%d\n"
, "!trylock", "/usr/src/sys/dev/pci/drm/i915/gt/intel_ggtt.c"
, 805); __builtin_expect(!!(__ret), 0); });
806
807 WARN_ON(__i915_vma_unbind(vma))({ int __ret = !!(__i915_vma_unbind(vma)); if (__ret) printf(
"WARNING %s failed at %s:%d\n", "__i915_vma_unbind(vma)", "/usr/src/sys/dev/pci/drm/i915/gt/intel_ggtt.c"
, 807); __builtin_expect(!!(__ret), 0); });
808 if (trylock)
809 i915_gem_object_unlock(obj);
810 }
811
812 if (drm_mm_node_allocated(&ggtt->error_capture))
813 drm_mm_remove_node(&ggtt->error_capture);
814 mutex_destroy(&ggtt->error_mutex);
815
816 ggtt_release_guc_top(ggtt);
817 intel_vgt_deballoon(ggtt);
818
819 ggtt->vm.cleanup(&ggtt->vm);
820
821 mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex);
822 i915_address_space_fini(&ggtt->vm);
823
824#ifdef notyet
825 arch_phys_wc_del(ggtt->mtrr);
826
827 if (ggtt->iomap.size)
828 io_mapping_fini(&ggtt->iomap);
829#endif
830}
831
832/**
833 * i915_ggtt_driver_release - Clean up GGTT hardware initialization
834 * @i915: i915 device
835 */
836void i915_ggtt_driver_release(struct drm_i915_privateinteldrm_softc *i915)
837{
838 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
839
840 fini_aliasing_ppgtt(ggtt);
841
842 intel_ggtt_fini_fences(ggtt);
843 ggtt_cleanup_hw(ggtt);
844}
845
846/**
847 * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after
848 * all free objects have been drained.
849 * @i915: i915 device
850 */
851void i915_ggtt_driver_late_release(struct drm_i915_privateinteldrm_softc *i915)
852{
853 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
854
855 GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1)({ __builtin_expect(!!(!!(kref_read(&ggtt->vm.resv_ref
) != 1)), 0); });
856 dma_resv_fini(&ggtt->vm._resv);
857}
858
859static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
860{
861 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT8;
862 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK0x3;
863 return snb_gmch_ctl << 20;
864}
865
866static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
867{
868 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT6;
869 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK0x3;
870 if (bdw_gmch_ctl)
871 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
872
873#ifdef CONFIG_X86_32
874 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
875 if (bdw_gmch_ctl > 4)
876 bdw_gmch_ctl = 4;
877#endif
878
879 return bdw_gmch_ctl << 20;
880}
881
882static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
883{
884 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT8;
885 gmch_ctrl &= SNB_GMCH_GGMS_MASK0x3;
886
887 if (gmch_ctrl)
888 return 1 << (20 + gmch_ctrl);
889
890 return 0;
891}
892
893static unsigned int gen6_gttmmadr_size(struct drm_i915_privateinteldrm_softc *i915)
894{
895 /*
896 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset
897 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset
898 */
899 GEM_BUG_ON(GRAPHICS_VER(i915) < 6)((void)0);
900 return (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) < 8) ? SZ_4M(4 << 20) : SZ_16M(16 << 20);
901}
902
903static unsigned int gen6_gttadr_offset(struct drm_i915_privateinteldrm_softc *i915)
904{
905 return gen6_gttmmadr_size(i915) / 2;
906}
907
908#ifdef __linux__
909
910static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
911{
912 struct drm_i915_privateinteldrm_softc *i915 = ggtt->vm.i915;
913 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
914 phys_addr_t phys_addr;
915 u32 pte_flags;
916 int ret;
917
918 GEM_WARN_ON(pci_resource_len(pdev, GTTMMADR_BAR) != gen6_gttmmadr_size(i915))({ __builtin_expect(!!(!!(pci_resource_len(pdev, 0) != gen6_gttmmadr_size
(i915))), 0); });
919 phys_addr = pci_resource_start(pdev, GTTMMADR_BAR0) + gen6_gttadr_offset(i915);
920
921 /*
922 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
923 * will be dropped. For WC mappings in general we have 64 byte burst
924 * writes when the WC buffer is flushed, so we can't use it, but have to
925 * resort to an uncached mapping. The WC issue is easily caught by the
926 * readback check when writing GTT PTE entries.
927 */
928 if (IS_GEN9_LP(i915)(((&(i915)->__runtime)->graphics.ip.ver) == 9 &&
((&(i915)->__info)->is_lp)) || GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 11)
929 ggtt->gsm = ioremap(phys_addr, size);
930 else
931 ggtt->gsm = ioremap_wc(phys_addr, size);
932 if (!ggtt->gsm) {
933 drm_err(&i915->drm, "Failed to map the ggtt page table\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to map the ggtt page table\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
934 return -ENOMEM12;
935 }
936
937 kref_init(&ggtt->vm.resv_ref);
938 ret = setup_scratch_page(&ggtt->vm);
939 if (ret) {
940 drm_err(&i915->drm, "Scratch setup failed\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Scratch setup failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
941 /* iounmap will also get called at remove, but meh */
942 iounmap(ggtt->gsm);
943 return ret;
944 }
945
946 pte_flags = 0;
947 if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))
948 pte_flags |= PTE_LM(1UL << (1));
949
950 ggtt->vm.scratch[0]->encode =
951 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0])(__px_dma(__builtin_choose_expr( __builtin_types_compatible_p
(typeof(ggtt->vm.scratch[0]), struct drm_i915_gem_object *
) || __builtin_types_compatible_p(typeof(ggtt->vm.scratch[
0]), const struct drm_i915_gem_object *), ({ struct drm_i915_gem_object
* __x = (struct drm_i915_gem_object *)(ggtt->vm.scratch[0
]); __x; }), __builtin_choose_expr( __builtin_types_compatible_p
(typeof(ggtt->vm.scratch[0]), struct i915_page_table *) ||
__builtin_types_compatible_p(typeof(ggtt->vm.scratch[0]),
const struct i915_page_table *), ({ struct i915_page_table *
__x = (struct i915_page_table *)(ggtt->vm.scratch[0]); __x
->base; }), __builtin_choose_expr( __builtin_types_compatible_p
(typeof(ggtt->vm.scratch[0]), struct i915_page_directory *
) || __builtin_types_compatible_p(typeof(ggtt->vm.scratch[
0]), const struct i915_page_directory *), ({ struct i915_page_directory
* __x = (struct i915_page_directory *)(ggtt->vm.scratch[0
]); __x->pt.base; }), (void)0))))),
952 I915_CACHE_NONE, pte_flags);
953
954 return 0;
955}
956
957#else
958
959static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
960{
961 struct drm_i915_privateinteldrm_softc *i915 = ggtt->vm.i915;
962 struct pci_dev *pdev = i915->drm.pdev;
Value stored to 'pdev' during its initialization is never read
963 phys_addr_t phys_addr;
964 bus_addr_t addr;
965 bus_size_t len;
966 pcireg_t type;
967 int flags;
968 u32 pte_flags;
969 int ret;
970
971 /* For Modern GENs the PTEs and register space are split in the BAR */
972 type = pci_mapreg_type(i915->pc, i915->tag, 0x10);
973 ret = -pci_mapreg_info(i915->pc, i915->tag, 0x10, type,
974 &addr, &len, NULL((void *)0));
975 if (ret)
976 return ret;
977
978 /*
979 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
980 * will be dropped. For WC mappings in general we have 64 byte burst
981 * writes when the WC buffer is flushed, so we can't use it, but have to
982 * resort to an uncached mapping. The WC issue is easily caught by the
983 * readback check when writing GTT PTE entries.
984 */
985 if (IS_GEN9_LP(i915)(((&(i915)->__runtime)->graphics.ip.ver) == 9 &&
((&(i915)->__info)->is_lp)) || GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 11)
986 flags = 0;
987 else
988 flags = BUS_SPACE_MAP_PREFETCHABLE0x0008;
989 ret = -bus_space_map(i915->bst, addr + len / 2, size,
990 flags | BUS_SPACE_MAP_LINEAR0x0002, &ggtt->gsm_bsh);
991 if (ret) {
992 drm_err(&i915->drm, "Failed to map the ggtt page table\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to map the ggtt page table\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
993 return ret;
994 }
995 ggtt->gsm = bus_space_vaddr(i915->bst, ggtt->gsm_bsh)((i915->bst)->vaddr((ggtt->gsm_bsh)));
996 ggtt->gsm_size = size;
997 if (!ggtt->gsm) {
998 DRM_ERROR("Failed to map the ggtt page table\n")__drm_err("Failed to map the ggtt page table\n");
999 return -ENOMEM12;
1000 }
1001
1002 kref_init(&ggtt->vm.resv_ref);
1003 ret = setup_scratch_page(&ggtt->vm);
1004 if (ret) {
1005 drm_err(&i915->drm, "Scratch setup failed\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Scratch setup failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
1006 /* iounmap will also get called at remove, but meh */
1007 bus_space_unmap(i915->bst, ggtt->gsm_bsh, size);
1008 return ret;
1009 }
1010
1011 pte_flags = 0;
1012 if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))
1013 pte_flags |= PTE_LM(1UL << (1));
1014
1015 ggtt->vm.scratch[0]->encode =
1016 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0])(__px_dma(__builtin_choose_expr( __builtin_types_compatible_p
(typeof(ggtt->vm.scratch[0]), struct drm_i915_gem_object *
) || __builtin_types_compatible_p(typeof(ggtt->vm.scratch[
0]), const struct drm_i915_gem_object *), ({ struct drm_i915_gem_object
* __x = (struct drm_i915_gem_object *)(ggtt->vm.scratch[0
]); __x; }), __builtin_choose_expr( __builtin_types_compatible_p
(typeof(ggtt->vm.scratch[0]), struct i915_page_table *) ||
__builtin_types_compatible_p(typeof(ggtt->vm.scratch[0]),
const struct i915_page_table *), ({ struct i915_page_table *
__x = (struct i915_page_table *)(ggtt->vm.scratch[0]); __x
->base; }), __builtin_choose_expr( __builtin_types_compatible_p
(typeof(ggtt->vm.scratch[0]), struct i915_page_directory *
) || __builtin_types_compatible_p(typeof(ggtt->vm.scratch[
0]), const struct i915_page_directory *), ({ struct i915_page_directory
* __x = (struct i915_page_directory *)(ggtt->vm.scratch[0
]); __x->pt.base; }), (void)0))))),
1017 I915_CACHE_NONE, pte_flags);
1018
1019 return 0;
1020}
1021
1022#endif
1023
1024static void gen6_gmch_remove(struct i915_address_space *vm)
1025{
1026 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
1027
1028#ifdef __linux__
1029 iounmap(ggtt->gsm);
1030#else
1031 bus_space_unmap(vm->i915->bst, ggtt->gsm_bsh, ggtt->gsm_size);
1032#endif
1033 free_scratch(vm);
1034}
1035
1036#ifdef __linux__
1037static struct resource pci_resource(struct pci_dev *pdev, int bar)
1038{
1039 return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar),{ .start = (pci_resource_start(pdev, bar)), .end = (pci_resource_start
(pdev, bar)) + (pci_resource_len(pdev, bar)) - 1, }
1040 pci_resource_len(pdev, bar)){ .start = (pci_resource_start(pdev, bar)), .end = (pci_resource_start
(pdev, bar)) + (pci_resource_len(pdev, bar)) - 1, };
1041}
1042#endif
1043
1044static int gen8_gmch_probe(struct i915_ggtt *ggtt)
1045{
1046 struct drm_i915_privateinteldrm_softc *i915 = ggtt->vm.i915;
1047 struct pci_dev *pdev = i915->drm.pdev;
1048 unsigned int size;
1049 u16 snb_gmch_ctl;
1050
1051 if (!HAS_LMEM(i915)((&(i915)->__runtime)->memory_regions & ((1UL <<
(INTEL_REGION_LMEM_0))))) {
1052#ifdef __linux__
1053 if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR2))
1054 return -ENXIO6;
1055
1056 ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR2);
1057 ggtt->mappable_end = resource_size(&ggtt->gmadr);
1058#else
1059 bus_addr_t base;
1060 bus_size_t sz;
1061 pcireg_t type;
1062 int err;
1063
1064 type = pci_mapreg_type(i915->pc, i915->tag, 0x18);
1065 err = -pci_mapreg_info(i915->pc, i915->tag, 0x18, type,
1066 &base, &sz, NULL((void *)0));
1067 if (err)
1068 return err;
1069 ggtt->gmadr.start = base;
1070 ggtt->mappable_end = sz;
1071#endif
1072 }
1073
1074 pci_read_config_word(pdev, SNB_GMCH_CTRL0x50, &snb_gmch_ctl);
1075 if (IS_CHERRYVIEW(i915)IS_PLATFORM(i915, INTEL_CHERRYVIEW))
1076 size = chv_get_total_gtt_size(snb_gmch_ctl);
1077 else
1078 size = gen8_get_total_gtt_size(snb_gmch_ctl);
1079
1080 ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1081 ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
1082 ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY(1UL << (5));
1083
1084 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE(1ULL << (12));
1085 ggtt->vm.cleanup = gen6_gmch_remove;
1086 ggtt->vm.insert_page = gen8_ggtt_insert_page;
1087 ggtt->vm.clear_range = nop_clear_range;
1088 if (intel_scanout_needs_vtd_wa(i915))
1089 ggtt->vm.clear_range = gen8_ggtt_clear_range;
1090
1091 ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
1092
1093 /*
1094 * Serialize GTT updates with aperture access on BXT if VT-d is on,
1095 * and always on CHV.
1096 */
1097 if (intel_vm_no_concurrent_access_wa(i915)) {
1098 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
1099 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL;
1100
1101 /*
1102 * Calling stop_machine() version of GGTT update function
1103 * at error capture/reset path will raise lockdep warning.
1104 * Allow calling gen8_ggtt_insert_* directly at reset path
1105 * which is safe from parallel GGTT updates.
1106 */
1107 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
1108 ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries;
1109
1110 ggtt->vm.bind_async_flags =
1111 I915_VMA_GLOBAL_BIND((int)(1UL << (10))) | I915_VMA_LOCAL_BIND((int)(1UL << (11)));
1112 }
1113
1114 ggtt->invalidate = gen8_ggtt_invalidate;
1115
1116 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;
1117 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
1118
1119 ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
1120
1121 setup_private_pat(ggtt->vm.gt->uncore);
1122
1123 return ggtt_probe_common(ggtt, size);
1124}
1125
1126static u64 snb_pte_encode(dma_addr_t addr,
1127 enum i915_cache_level level,
1128 u32 flags)
1129{
1130 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr)((addr) | (((addr) >> 28) & 0xff0)) | GEN6_PTE_VALID((u32)((1UL << (0)) + 0));
1131
1132 switch (level) {
1133 case I915_CACHE_L3_LLC:
1134 case I915_CACHE_LLC:
1135 pte |= GEN6_PTE_CACHE_LLC(2 << 1);
1136 break;
1137 case I915_CACHE_NONE:
1138 pte |= GEN6_PTE_UNCACHED(1 << 1);
1139 break;
1140 default:
1141 MISSING_CASE(level)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "level", (long)(level)); __builtin_expect(!!(__ret), 0); });
1142 }
1143
1144 return pte;
1145}
1146
1147static u64 ivb_pte_encode(dma_addr_t addr,
1148 enum i915_cache_level level,
1149 u32 flags)
1150{
1151 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr)((addr) | (((addr) >> 28) & 0xff0)) | GEN6_PTE_VALID((u32)((1UL << (0)) + 0));
1152
1153 switch (level) {
1154 case I915_CACHE_L3_LLC:
1155 pte |= GEN7_PTE_CACHE_L3_LLC(3 << 1);
1156 break;
1157 case I915_CACHE_LLC:
1158 pte |= GEN6_PTE_CACHE_LLC(2 << 1);
1159 break;
1160 case I915_CACHE_NONE:
1161 pte |= GEN6_PTE_UNCACHED(1 << 1);
1162 break;
1163 default:
1164 MISSING_CASE(level)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "level", (long)(level)); __builtin_expect(!!(__ret), 0); });
1165 }
1166
1167 return pte;
1168}
1169
1170static u64 byt_pte_encode(dma_addr_t addr,
1171 enum i915_cache_level level,
1172 u32 flags)
1173{
1174 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr)((addr) | (((addr) >> 28) & 0xff0)) | GEN6_PTE_VALID((u32)((1UL << (0)) + 0));
1175
1176 if (!(flags & PTE_READ_ONLY(1UL << (0))))
1177 pte |= BYT_PTE_WRITEABLE((u32)((1UL << (1)) + 0));
1178
1179 if (level != I915_CACHE_NONE)
1180 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES((u32)((1UL << (2)) + 0));
1181
1182 return pte;
1183}
1184
1185static u64 hsw_pte_encode(dma_addr_t addr,
1186 enum i915_cache_level level,
1187 u32 flags)
1188{
1189 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr)((addr) | (((addr) >> 28) & 0x7f0)) | GEN6_PTE_VALID((u32)((1UL << (0)) + 0));
1190
1191 if (level != I915_CACHE_NONE)
1192 pte |= HSW_WB_LLC_AGE3((((0x2) & 0x7) << 1) | (((0x2) & 0x8) <<
(11 - 3)));
1193
1194 return pte;
1195}
1196
1197static u64 iris_pte_encode(dma_addr_t addr,
1198 enum i915_cache_level level,
1199 u32 flags)
1200{
1201 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr)((addr) | (((addr) >> 28) & 0x7f0)) | GEN6_PTE_VALID((u32)((1UL << (0)) + 0));
1202
1203 switch (level) {
1204 case I915_CACHE_NONE:
1205 break;
1206 case I915_CACHE_WT:
1207 pte |= HSW_WT_ELLC_LLC_AGE3((((0x7) & 0x7) << 1) | (((0x7) & 0x8) <<
(11 - 3)));
1208 break;
1209 default:
1210 pte |= HSW_WB_ELLC_LLC_AGE3((((0x8) & 0x7) << 1) | (((0x8) & 0x8) <<
(11 - 3)));
1211 break;
1212 }
1213
1214 return pte;
1215}
1216
1217static int gen6_gmch_probe(struct i915_ggtt *ggtt)
1218{
1219 struct drm_i915_privateinteldrm_softc *i915 = ggtt->vm.i915;
1220 struct pci_dev *pdev = i915->drm.pdev;
1221 unsigned int size;
1222 u16 snb_gmch_ctl;
1223
1224#ifdef __linux__
1225 if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR2))
1226 return -ENXIO6;
1227
1228 ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR2);
1229 ggtt->mappable_end = resource_size(&ggtt->gmadr);
1230#else
1231{
1232 bus_addr_t base;
1233 bus_size_t sz;
1234 pcireg_t type;
1235 int err;
1236
1237 type = pci_mapreg_type(i915->pc, i915->tag, 0x18);
1238 err = -pci_mapreg_info(i915->pc, i915->tag, 0x18, type,
1239 &base, &sz, NULL((void *)0));
1240 if (err)
1241 return err;
1242 ggtt->gmadr.start = base;
1243 ggtt->mappable_end = sz;
1244}
1245#endif
1246
1247 /*
1248 * 64/512MB is the current min/max we actually know of, but this is
1249 * just a coarse sanity check.
1250 */
1251 if (ggtt->mappable_end < (64 << 20) ||
1252 ggtt->mappable_end > (512 << 20)) {
1253 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Unknown GMADR size (%pa)\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , &ggtt
->mappable_end)
1254 &ggtt->mappable_end)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Unknown GMADR size (%pa)\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , &ggtt
->mappable_end);
1255 return -ENXIO6;
1256 }
1257
1258 pci_read_config_word(pdev, SNB_GMCH_CTRL0x50, &snb_gmch_ctl);
1259
1260 size = gen6_get_total_gtt_size(snb_gmch_ctl);
1261 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE(1ULL << (12));
1262
1263 ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1264 ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
1265
1266 ggtt->vm.clear_range = nop_clear_range;
1267 if (!HAS_FULL_PPGTT(i915)(((&(i915)->__runtime)->ppgtt_type) >= INTEL_PPGTT_FULL
) || intel_scanout_needs_vtd_wa(i915))
1268 ggtt->vm.clear_range = gen6_ggtt_clear_range;
1269 ggtt->vm.insert_page = gen6_ggtt_insert_page;
1270 ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
1271 ggtt->vm.cleanup = gen6_gmch_remove;
1272
1273 ggtt->invalidate = gen6_ggtt_invalidate;
1274
1275 if (HAS_EDRAM(i915)((i915)->edram_size_mb))
1276 ggtt->vm.pte_encode = iris_pte_encode;
1277 else if (IS_HASWELL(i915)IS_PLATFORM(i915, INTEL_HASWELL))
1278 ggtt->vm.pte_encode = hsw_pte_encode;
1279 else if (IS_VALLEYVIEW(i915)IS_PLATFORM(i915, INTEL_VALLEYVIEW))
1280 ggtt->vm.pte_encode = byt_pte_encode;
1281 else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 7)
1282 ggtt->vm.pte_encode = ivb_pte_encode;
1283 else
1284 ggtt->vm.pte_encode = snb_pte_encode;
1285
1286 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;
1287 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
1288
1289 return ggtt_probe_common(ggtt, size);
1290}
1291
1292static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
1293{
1294 struct drm_i915_privateinteldrm_softc *i915 = gt->i915;
1295 int ret;
1296
1297 ggtt->vm.gt = gt;
1298 ggtt->vm.i915 = i915;
1299#ifdef notyet
1300 ggtt->vm.dma = i915->drm.dev;
1301#endif
1302 dma_resv_init(&ggtt->vm._resv);
1303
1304 if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 8)
1305 ret = gen8_gmch_probe(ggtt);
1306 else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 6)
1307 ret = gen6_gmch_probe(ggtt);
1308 else
1309 ret = intel_ggtt_gmch_probe(ggtt);
1310
1311 if (ret) {
1312 dma_resv_fini(&ggtt->vm._resv);
1313 return ret;
1314 }
1315
1316 if ((ggtt->vm.total - 1) >> 32) {
1317 drm_err(&i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "We never expected a Global GTT with more than 32bits"
" of address space! Found %lldM!\n", ({struct cpu_info *__ci
; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , ggtt->vm.total >> 20)
1318 "We never expected a Global GTT with more than 32bits"printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "We never expected a Global GTT with more than 32bits"
" of address space! Found %lldM!\n", ({struct cpu_info *__ci
; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , ggtt->vm.total >> 20)
1319 " of address space! Found %lldM!\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "We never expected a Global GTT with more than 32bits"
" of address space! Found %lldM!\n", ({struct cpu_info *__ci
; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , ggtt->vm.total >> 20)
1320 ggtt->vm.total >> 20)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "We never expected a Global GTT with more than 32bits"
" of address space! Found %lldM!\n", ({struct cpu_info *__ci
; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , ggtt->vm.total >> 20);
1321 ggtt->vm.total = 1ULL << 32;
1322 ggtt->mappable_end =
1323 min_t(u64, ggtt->mappable_end, ggtt->vm.total)({ u64 __min_a = (ggtt->mappable_end); u64 __min_b = (ggtt
->vm.total); __min_a < __min_b ? __min_a : __min_b; });
1324 }
1325
1326 if (ggtt->mappable_end > ggtt->vm.total) {
1327 drm_err(&i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "mappable aperture extends past end of GGTT,"
" aperture=%pa, total=%llx\n", ({struct cpu_info *__ci; asm volatile
("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct
cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid
, __func__ , &ggtt->mappable_end, ggtt->vm.total)
1328 "mappable aperture extends past end of GGTT,"printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "mappable aperture extends past end of GGTT,"
" aperture=%pa, total=%llx\n", ({struct cpu_info *__ci; asm volatile
("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct
cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid
, __func__ , &ggtt->mappable_end, ggtt->vm.total)
1329 " aperture=%pa, total=%llx\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "mappable aperture extends past end of GGTT,"
" aperture=%pa, total=%llx\n", ({struct cpu_info *__ci; asm volatile
("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct
cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid
, __func__ , &ggtt->mappable_end, ggtt->vm.total)
1330 &ggtt->mappable_end, ggtt->vm.total)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "mappable aperture extends past end of GGTT,"
" aperture=%pa, total=%llx\n", ({struct cpu_info *__ci; asm volatile
("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct
cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid
, __func__ , &ggtt->mappable_end, ggtt->vm.total);
1331 ggtt->mappable_end = ggtt->vm.total;
1332 }
1333
1334 /* GMADR is the PCI mmio aperture into the global GTT. */
1335 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20)__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "GGTT size = %lluM\n"
, ggtt->vm.total >> 20);
1336 drm_dbg(&i915->drm, "GMADR size = %lluM\n",__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "GMADR size = %lluM\n"
, (u64)ggtt->mappable_end >> 20)
1337 (u64)ggtt->mappable_end >> 20)__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "GMADR size = %lluM\n"
, (u64)ggtt->mappable_end >> 20);
1338 drm_dbg(&i915->drm, "DSM size = %lluM\n",__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "DSM size = %lluM\n"
, (u64)resource_size(&intel_graphics_stolen_res) >>
20)
1339 (u64)resource_size(&intel_graphics_stolen_res) >> 20)__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "DSM size = %lluM\n"
, (u64)resource_size(&intel_graphics_stolen_res) >>
20);
1340
1341 return 0;
1342}
1343
1344/**
1345 * i915_ggtt_probe_hw - Probe GGTT hardware location
1346 * @i915: i915 device
1347 */
1348int i915_ggtt_probe_hw(struct drm_i915_privateinteldrm_softc *i915)
1349{
1350 int ret;
1351
1352 ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
1353 if (ret)
1354 return ret;
1355
1356 if (i915_vtd_active(i915))
1357 drm_info(&i915->drm, "VT-d active for gfx access\n")do { } while(0);
1358
1359 return 0;
1360}
1361
1362int i915_ggtt_enable_hw(struct drm_i915_privateinteldrm_softc *i915)
1363{
1364 if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) < 6)
1365 return intel_ggtt_gmch_enable_hw(i915);
1366
1367 return 0;
1368}
1369
1370void i915_ggtt_enable_guc(struct i915_ggtt *ggtt)
1371{
1372 GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate)((void)0);
1373
1374 ggtt->invalidate = guc_ggtt_invalidate;
1375
1376 ggtt->invalidate(ggtt);
1377}
1378
1379void i915_ggtt_disable_guc(struct i915_ggtt *ggtt)
1380{
1381 /* XXX Temporary pardon for error unload */
1382 if (ggtt->invalidate == gen8_ggtt_invalidate)
1383 return;
1384
1385 /* We should only be called after i915_ggtt_enable_guc() */
1386 GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate)((void)0);
1387
1388 ggtt->invalidate = gen8_ggtt_invalidate;
1389
1390 ggtt->invalidate(ggtt);
1391}
1392
1393/**
1394 * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM
1395 * @vm: The VM to restore the mappings for
1396 *
1397 * Restore the memory mappings for all objects mapped to HW via the GGTT or a
1398 * DPT page table.
1399 *
1400 * Returns %true if restoring the mapping for any object that was in a write
1401 * domain before suspend.
1402 */
1403bool_Bool i915_ggtt_resume_vm(struct i915_address_space *vm)
1404{
1405 struct i915_vma *vma;
1406 bool_Bool write_domain_objs = false0;
1407 bool_Bool retained_ptes;
1408
1409 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt)({ int __ret = !!((!vm->is_ggtt && !vm->is_dpt)
); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
vm->i915->drm))->dev), "", "drm_WARN_ON(" "!vm->is_ggtt && !vm->is_dpt"
")"); __builtin_expect(!!(__ret), 0); });
1410
1411 /*
1412 * First fill our portion of the GTT with scratch pages if
1413 * they were not retained across suspend.
1414 */
1415 retained_ptes = suspend_retains_ptes(vm) &&
1416 !i915_vm_to_ggtt(vm)->pte_lost &&
1417 !GEM_WARN_ON(i915_vm_to_ggtt(vm)->probed_pte != read_last_pte(vm))({ __builtin_expect(!!(!!(i915_vm_to_ggtt(vm)->probed_pte !=
read_last_pte(vm))), 0); });
1418
1419 if (!retained_ptes)
1420 vm->clear_range(vm, 0, vm->total);
1421
1422 /* clflush objects bound into the GGTT and rebind them. */
1423 list_for_each_entry(vma, &vm->bound_list, vm_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link
) *__mptr = ((&vm->bound_list)->next); (__typeof(*
vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), vm_link
) );}); &vma->vm_link != (&vm->bound_list); vma
= ({ const __typeof( ((__typeof(*vma) *)0)->vm_link ) *__mptr
= (vma->vm_link.next); (__typeof(*vma) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*vma), vm_link) );})) {
1424 struct drm_i915_gem_object *obj = vma->obj;
1425 unsigned int was_bound =
1426 atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*(
&vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer
(); __tmp; }) & I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11))));
1427
1428 GEM_BUG_ON(!was_bound)((void)0);
1429 if (!retained_ptes) {
1430 /*
1431 * Clear the bound flags of the vma resource to allow
1432 * ptes to be repopulated.
1433 */
1434 vma->resource->bound_flags = 0;
1435 vma->ops->bind_vma(vm, NULL((void *)0), vma->resource,
1436 obj ? obj->cache_level : 0,
1437 was_bound);
1438 }
1439 if (obj) { /* only used during resume => exclusive access */
1440 write_domain_objs |= fetch_and_zero(&obj->write_domain)({ typeof(*&obj->write_domain) __T = *(&obj->write_domain
); *(&obj->write_domain) = (typeof(*&obj->write_domain
))0; __T; });
1441 obj->read_domains |= I915_GEM_DOMAIN_GTT0x00000040;
1442 }
1443 }
1444
1445 return write_domain_objs;
1446}
1447
1448void i915_ggtt_resume(struct i915_ggtt *ggtt)
1449{
1450 bool_Bool flush;
1451
1452 intel_gt_check_and_clear_faults(ggtt->vm.gt);
1453
1454 flush = i915_ggtt_resume_vm(&ggtt->vm);
1455
1456 ggtt->invalidate(ggtt);
1457
1458 if (flush)
1459 wbinvd_on_all_cpus();
1460
1461 if (GRAPHICS_VER(ggtt->vm.i915)((&(ggtt->vm.i915)->__runtime)->graphics.ip.ver) >= 8)
1462 setup_private_pat(ggtt->vm.gt->uncore);
1463
1464 intel_ggtt_restore_fences(ggtt);
1465}
1466
1467void i915_ggtt_mark_pte_lost(struct drm_i915_privateinteldrm_softc *i915, bool_Bool val)
1468{
1469 to_gt(i915)->ggtt->pte_lost = val;
1470}