Bug Summary

File:dev/pci/drm/i915/i915_vma.c
Warning:line 1216, column 2
Value stored to 'node' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name i915_vma.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/i915/i915_vma.c
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/sched/mm.h>
26#include <drm/drm_gem.h>
27
28#include "display/intel_frontbuffer.h"
29
30#include "gt/intel_engine.h"
31#include "gt/intel_engine_heartbeat.h"
32#include "gt/intel_gt.h"
33#include "gt/intel_gt_requests.h"
34
35#include "i915_drv.h"
36#include "i915_globals.h"
37#include "i915_sw_fence_work.h"
38#include "i915_trace.h"
39#include "i915_vma.h"
40
41#include <dev/pci/agpvar.h>
42
43static struct i915_global_vma {
44 struct i915_global base;
45#ifdef __linux__
46 struct kmem_cache *slab_vmas;
47#else
48 struct pool slab_vmas;
49#endif
50} global;
51
52struct i915_vma *i915_vma_alloc(void)
53{
54#ifdef __linux__
55 return kmem_cache_zalloc(global.slab_vmas, GFP_KERNEL(0x0001 | 0x0004));
56#else
57 return pool_get(&global.slab_vmas, PR_WAITOK0x0001 | PR_ZERO0x0008);
58#endif
59}
60
61void i915_vma_free(struct i915_vma *vma)
62{
63#ifdef __linux__
64 return kmem_cache_free(global.slab_vmas, vma);
65#else
66 pool_put(&global.slab_vmas, vma);
67#endif
68}
69
70#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM)0 && IS_ENABLED(CONFIG_DRM_DEBUG_MM)0
71
72#include <linux/stackdepot.h>
73
74static void vma_print_allocator(struct i915_vma *vma, const char *reason)
75{
76 unsigned long *entries;
77 unsigned int nr_entries;
78 char buf[512];
79
80 if (!vma->node.stack) {
81 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: unknown owner\n"
, vma->node.start, vma->node.size, reason)
82 vma->node.start, vma->node.size, reason)__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: unknown owner\n"
, vma->node.start, vma->node.size, reason)
;
83 return;
84 }
85
86 nr_entries = stack_depot_fetch(vma->node.stack, &entries);
87 stack_trace_snprint(buf, sizeof(buf), entries, nr_entries, 0);
88 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: inserted at %s\n"
, vma->node.start, vma->node.size, reason, buf)
89 vma->node.start, vma->node.size, reason, buf)__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: inserted at %s\n"
, vma->node.start, vma->node.size, reason, buf)
;
90}
91
92#else
93
94static void vma_print_allocator(struct i915_vma *vma, const char *reason)
95{
96}
97
98#endif
99
100static inline struct i915_vma *active_to_vma(struct i915_active *ref)
101{
102 return container_of(ref, typeof(struct i915_vma), active)({ const __typeof( ((typeof(struct i915_vma) *)0)->active )
*__mptr = (ref); (typeof(struct i915_vma) *)( (char *)__mptr
- __builtin_offsetof(typeof(struct i915_vma), active) );})
;
103}
104
105static int __i915_vma_active(struct i915_active *ref)
106{
107 return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT2;
108}
109
110__i915_active_call__attribute__((__aligned__(4)))
111static void __i915_vma_retire(struct i915_active *ref)
112{
113 i915_vma_put(active_to_vma(ref));
114}
115
116static struct i915_vma *
117vma_create(struct drm_i915_gem_object *obj,
118 struct i915_address_space *vm,
119 const struct i915_ggtt_view *view)
120{
121 struct i915_vma *pos = ERR_PTR(-E2BIG7);
122 struct i915_vma *vma;
123 struct rb_node *rb, **p;
124
125 /* The aliasing_ppgtt should never be used directly! */
126 GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm)((void)0);
127
128 vma = i915_vma_alloc();
129 if (vma == NULL((void *)0))
130 return ERR_PTR(-ENOMEM12);
131
132 kref_init(&vma->ref);
133 rw_init(&vma->pages_mutex, "vmapg")_rw_init_flags(&vma->pages_mutex, "vmapg", 0, ((void *
)0))
;
134 vma->vm = i915_vm_get(vm);
135 vma->ops = &vm->vma_ops;
136 vma->obj = obj;
137 vma->resv = obj->base.resv;
138 vma->size = obj->base.size;
139 vma->display_alignment = I915_GTT_MIN_ALIGNMENT(1ULL << (12));
140
141 i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire)do { static struct lock_class_key __mkey; static struct lock_class_key
__wkey; __i915_active_init(&vma->active, __i915_vma_active
, __i915_vma_retire, &__mkey, &__wkey); } while (0)
;
142
143#ifdef notyet
144 /* Declare ourselves safe for use inside shrinkers */
145 if (IS_ENABLED(CONFIG_LOCKDEP)0) {
146 fs_reclaim_acquire(GFP_KERNEL(0x0001 | 0x0004));
147 might_lock(&vma->active.mutex);
148 fs_reclaim_release(GFP_KERNEL(0x0001 | 0x0004));
149 }
150#endif
151
152 INIT_LIST_HEAD(&vma->closed_link);
153
154 if (view && view->type != I915_GGTT_VIEW_NORMAL) {
155 vma->ggtt_view = *view;
156 if (view->type == I915_GGTT_VIEW_PARTIAL) {
157 GEM_BUG_ON(range_overflows_t(u64,((void)0)
158 view->partial.offset,((void)0)
159 view->partial.size,((void)0)
160 obj->base.size >> PAGE_SHIFT))((void)0);
161 vma->size = view->partial.size;
162 vma->size <<= PAGE_SHIFT12;
163 GEM_BUG_ON(vma->size > obj->base.size)((void)0);
164 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
165 vma->size = intel_rotation_info_size(&view->rotated);
166 vma->size <<= PAGE_SHIFT12;
167 } else if (view->type == I915_GGTT_VIEW_REMAPPED) {
168 vma->size = intel_remapped_info_size(&view->remapped);
169 vma->size <<= PAGE_SHIFT12;
170 }
171 }
172
173 if (unlikely(vma->size > vm->total)__builtin_expect(!!(vma->size > vm->total), 0))
174 goto err_vma;
175
176 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE))((void)0);
177
178 spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
179
180 if (i915_is_ggtt(vm)((vm)->is_ggtt)) {
181 if (unlikely(overflows_type(vma->size, u32))__builtin_expect(!!((sizeof(vma->size) > sizeof(u32) &&
(vma->size) >> (8 * sizeof(u32)))), 0)
)
182 goto err_unlock;
183
184 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size,
185 i915_gem_object_get_tiling(obj),
186 i915_gem_object_get_stride(obj));
187 if (unlikely(vma->fence_size < vma->size || /* overflow */__builtin_expect(!!(vma->fence_size < vma->size || vma
->fence_size > vm->total), 0)
188 vma->fence_size > vm->total)__builtin_expect(!!(vma->fence_size < vma->size || vma
->fence_size > vm->total), 0)
)
189 goto err_unlock;
190
191 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT))((void)0);
192
193 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size,
194 i915_gem_object_get_tiling(obj),
195 i915_gem_object_get_stride(obj));
196 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment))((void)0);
197
198 __set_bit(I915_VMA_GGTT_BIT14, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
199 }
200
201 rb = NULL((void *)0);
202 p = &obj->vma.tree.rb_node;
203 while (*p) {
204 long cmp;
205
206 rb = *p;
207 pos = rb_entry(rb, struct i915_vma, obj_node)({ const __typeof( ((struct i915_vma *)0)->obj_node ) *__mptr
= (rb); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof
(struct i915_vma, obj_node) );})
;
208
209 /*
210 * If the view already exists in the tree, another thread
211 * already created a matching vma, so return the older instance
212 * and dispose of ours.
213 */
214 cmp = i915_vma_compare(pos, vm, view);
215 if (cmp < 0)
216 p = &rb->rb_right__entry.rbe_right;
217 else if (cmp > 0)
218 p = &rb->rb_left__entry.rbe_left;
219 else
220 goto err_unlock;
221 }
222 rb_link_node(&vma->obj_node, rb, p);
223 rb_insert_color(&vma->obj_node, &obj->vma.tree)linux_root_RB_INSERT_COLOR((struct linux_root *)(&obj->
vma.tree), (&vma->obj_node))
;
224
225 if (i915_vma_is_ggtt(vma))
226 /*
227 * We put the GGTT vma at the start of the vma-list, followed
228 * by the ppGGTT vma. This allows us to break early when
229 * iterating over only the GGTT vma for an object, see
230 * for_each_ggtt_vma()
231 */
232 list_add(&vma->obj_link, &obj->vma.list);
233 else
234 list_add_tail(&vma->obj_link, &obj->vma.list);
235
236 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
237
238 return vma;
239
240err_unlock:
241 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
242err_vma:
243 i915_vm_put(vm);
244 i915_vma_free(vma);
245 return pos;
246}
247
248static struct i915_vma *
249vma_lookup(struct drm_i915_gem_object *obj,
250 struct i915_address_space *vm,
251 const struct i915_ggtt_view *view)
252{
253 struct rb_node *rb;
254
255 rb = obj->vma.tree.rb_node;
256 while (rb) {
257 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node)({ const __typeof( ((struct i915_vma *)0)->obj_node ) *__mptr
= (rb); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof
(struct i915_vma, obj_node) );})
;
258 long cmp;
259
260 cmp = i915_vma_compare(vma, vm, view);
261 if (cmp == 0)
262 return vma;
263
264 if (cmp < 0)
265 rb = rb->rb_right__entry.rbe_right;
266 else
267 rb = rb->rb_left__entry.rbe_left;
268 }
269
270 return NULL((void *)0);
271}
272
273/**
274 * i915_vma_instance - return the singleton instance of the VMA
275 * @obj: parent &struct drm_i915_gem_object to be mapped
276 * @vm: address space in which the mapping is located
277 * @view: additional mapping requirements
278 *
279 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
280 * the same @view characteristics. If a match is not found, one is created.
281 * Once created, the VMA is kept until either the object is freed, or the
282 * address space is closed.
283 *
284 * Returns the vma, or an error pointer.
285 */
286struct i915_vma *
287i915_vma_instance(struct drm_i915_gem_object *obj,
288 struct i915_address_space *vm,
289 const struct i915_ggtt_view *view)
290{
291 struct i915_vma *vma;
292
293 GEM_BUG_ON(view && !i915_is_ggtt(vm))((void)0);
294 GEM_BUG_ON(!atomic_read(&vm->open))((void)0);
295
296 spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
297 vma = vma_lookup(obj, vm, view);
298 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
299
300 /* vma_create() will resolve the race if another creates the vma */
301 if (unlikely(!vma)__builtin_expect(!!(!vma), 0))
302 vma = vma_create(obj, vm, view);
303
304 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view))((void)0);
305 return vma;
306}
307
308struct i915_vma_work {
309 struct dma_fence_work base;
310 struct i915_address_space *vm;
311 struct i915_vm_pt_stash stash;
312 struct i915_vma *vma;
313 struct drm_i915_gem_object *pinned;
314 struct i915_sw_dma_fence_cb cb;
315 enum i915_cache_level cache_level;
316 unsigned int flags;
317};
318
319static int __vma_bind(struct dma_fence_work *work)
320{
321 struct i915_vma_work *vw = container_of(work, typeof(*vw), base)({ const __typeof( ((typeof(*vw) *)0)->base ) *__mptr = (work
); (typeof(*vw) *)( (char *)__mptr - __builtin_offsetof(typeof
(*vw), base) );})
;
322 struct i915_vma *vma = vw->vma;
323
324 vma->ops->bind_vma(vw->vm, &vw->stash,
325 vma, vw->cache_level, vw->flags);
326 return 0;
327}
328
329static void __vma_release(struct dma_fence_work *work)
330{
331 struct i915_vma_work *vw = container_of(work, typeof(*vw), base)({ const __typeof( ((typeof(*vw) *)0)->base ) *__mptr = (work
); (typeof(*vw) *)( (char *)__mptr - __builtin_offsetof(typeof
(*vw), base) );})
;
332
333 if (vw->pinned) {
334 __i915_gem_object_unpin_pages(vw->pinned);
335 i915_gem_object_put(vw->pinned);
336 }
337
338 i915_vm_free_pt_stash(vw->vm, &vw->stash);
339 i915_vm_put(vw->vm);
340}
341
342static const struct dma_fence_work_ops bind_ops = {
343 .name = "bind",
344 .work = __vma_bind,
345 .release = __vma_release,
346};
347
348struct i915_vma_work *i915_vma_work(void)
349{
350 struct i915_vma_work *vw;
351
352 vw = kzalloc(sizeof(*vw), GFP_KERNEL(0x0001 | 0x0004));
353 if (!vw)
354 return NULL((void *)0);
355
356 dma_fence_work_init(&vw->base, &bind_ops);
357 vw->base.dma.error = -EAGAIN35; /* disable the worker by default */
358
359 return vw;
360}
361
362int i915_vma_wait_for_bind(struct i915_vma *vma)
363{
364 int err = 0;
365
366 if (rcu_access_pointer(vma->active.excl.fence)(vma->active.excl.fence)) {
367 struct dma_fence *fence;
368
369 rcu_read_lock();
370 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence);
371 rcu_read_unlock();
372 if (fence) {
373 err = dma_fence_wait(fence, MAX_SCHEDULE_TIMEOUT(0x7fffffff));
374 dma_fence_put(fence);
375 }
376 }
377
378 return err;
379}
380
381/**
382 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
383 * @vma: VMA to map
384 * @cache_level: mapping cache level
385 * @flags: flags like global or local mapping
386 * @work: preallocated worker for allocating and binding the PTE
387 *
388 * DMA addresses are taken from the scatter-gather table of this object (or of
389 * this VMA in case of non-default GGTT views) and PTE entries set up.
390 * Note that DMA addresses are also the only part of the SG table we care about.
391 */
392int i915_vma_bind(struct i915_vma *vma,
393 enum i915_cache_level cache_level,
394 u32 flags,
395 struct i915_vma_work *work)
396{
397 u32 bind_flags;
398 u32 vma_flags;
399
400 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0);
401 GEM_BUG_ON(vma->size > vma->node.size)((void)0);
402
403 if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,({ ((void)0); 0; })
404 vma->node.size,({ ((void)0); 0; })
405 vma->vm->total))({ ((void)0); 0; }))
406 return -ENODEV19;
407
408 if (GEM_DEBUG_WARN_ON(!flags)({ ((void)0); 0; }))
409 return -EINVAL22;
410
411 bind_flags = flags;
412 bind_flags &= I915_VMA_GLOBAL_BIND((int)(1UL << (10))) | I915_VMA_LOCAL_BIND((int)(1UL << (11)));
413
414 vma_flags = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*(
&vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer
(); __tmp; })
;
415 vma_flags &= I915_VMA_GLOBAL_BIND((int)(1UL << (10))) | I915_VMA_LOCAL_BIND((int)(1UL << (11)));
416
417 bind_flags &= ~vma_flags;
418 if (bind_flags == 0)
419 return 0;
420
421 GEM_BUG_ON(!vma->pages)((void)0);
422
423 trace_i915_vma_bind(vma, bind_flags);
424 if (work && bind_flags & vma->vm->bind_async_flags) {
425 struct dma_fence *prev;
426
427 work->vma = vma;
428 work->cache_level = cache_level;
429 work->flags = bind_flags;
430
431 /*
432 * Note we only want to chain up to the migration fence on
433 * the pages (not the object itself). As we don't track that,
434 * yet, we have to use the exclusive fence instead.
435 *
436 * Also note that we do not want to track the async vma as
437 * part of the obj->resv->excl_fence as it only affects
438 * execution and not content or object's backing store lifetime.
439 */
440 prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
441 if (prev) {
442 __i915_sw_fence_await_dma_fence(&work->base.chain,
443 prev,
444 &work->cb);
445 dma_fence_put(prev);
446 }
447
448 work->base.dma.error = 0; /* enable the queue_work() */
449
450 if (vma->obj) {
451 __i915_gem_object_pin_pages(vma->obj);
452 work->pinned = i915_gem_object_get(vma->obj);
453 }
454 } else {
455 vma->ops->bind_vma(vma->vm, NULL((void *)0), vma, cache_level, bind_flags);
456 }
457
458 atomic_or(bind_flags, &vma->flags)x86_atomic_setbits_u32(&vma->flags, bind_flags);
459 return 0;
460}
461
462void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
463{
464 void __iomem *ptr;
465 int err;
466
467 if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))({ __builtin_expect(!!(!!(!i915_vma_is_map_and_fenceable(vma)
)), 0); })
) {
468 err = -ENODEV19;
469 goto err;
470 }
471
472 GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0);
473 GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))((void)0);
474
475 ptr = READ_ONCE(vma->iomap)({ typeof(vma->iomap) __tmp = *(volatile typeof(vma->iomap
) *)&(vma->iomap); membar_datadep_consumer(); __tmp; }
)
;
476 if (ptr == NULL((void *)0)) {
477#ifdef __linux__
478 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
479 vma->node.start,
480 vma->node.size);
481#else
482 struct drm_i915_privateinteldrm_softc *dev_priv = vma->vm->i915;
483 err = agp_map_subregion(dev_priv->agph, vma->node.start,
484 vma->node.size, &vma->bsh);
485 if (err) {
486 err = -err;
487 goto err;
488 }
489 ptr = bus_space_vaddr(dev_priv->bst, vma->bsh)((dev_priv->bst)->vaddr((vma->bsh)));
490#endif
491 if (ptr == NULL((void *)0)) {
492 err = -ENOMEM12;
493 goto err;
494 }
495
496 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))__builtin_expect(!!(__sync_val_compare_and_swap(&vma->
iomap, ((void *)0), ptr)), 0)
) {
497#ifdef __linux__
498 io_mapping_unmap(ptr);
499#endif
500 ptr = vma->iomap;
501 }
502 }
503
504 __i915_vma_pin(vma);
505
506 err = i915_vma_pin_fence(vma);
507 if (err)
508 goto err_unpin;
509
510 i915_vma_set_ggtt_write(vma);
511
512 /* NB Access through the GTT requires the device to be awake. */
513 return ptr;
514
515err_unpin:
516 __i915_vma_unpin(vma);
517err:
518 return IO_ERR_PTR(err)((void *)ERR_PTR(err));
519}
520
521void i915_vma_flush_writes(struct i915_vma *vma)
522{
523 if (i915_vma_unset_ggtt_write(vma))
524 intel_gt_flush_ggtt_writes(vma->vm->gt);
525}
526
527void i915_vma_unpin_iomap(struct i915_vma *vma)
528{
529 GEM_BUG_ON(vma->iomap == NULL)((void)0);
530
531 i915_vma_flush_writes(vma);
532
533 i915_vma_unpin_fence(vma);
534 i915_vma_unpin(vma);
535}
536
537void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
538{
539 struct i915_vma *vma;
540 struct drm_i915_gem_object *obj;
541
542 vma = fetch_and_zero(p_vma)({ typeof(*p_vma) __T = *(p_vma); *(p_vma) = (typeof(*p_vma))
0; __T; })
;
543 if (!vma)
544 return;
545
546 obj = vma->obj;
547 GEM_BUG_ON(!obj)((void)0);
548
549 i915_vma_unpin(vma);
550
551 if (flags & I915_VMA_RELEASE_MAP(1UL << (0)))
552 i915_gem_object_unpin_map(obj);
553
554 i915_gem_object_put(obj);
555}
556
557bool_Bool i915_vma_misplaced(const struct i915_vma *vma,
558 u64 size, u64 alignment, u64 flags)
559{
560 if (!drm_mm_node_allocated(&vma->node))
561 return false0;
562
563 if (test_bit(I915_VMA_ERROR_BIT13, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags)))
564 return true1;
565
566 if (vma->node.size < size)
567 return true1;
568
569 GEM_BUG_ON(alignment && !is_power_of_2(alignment))((void)0);
570 if (alignment && !IS_ALIGNED(vma->node.start, alignment)(((vma->node.start) & ((alignment) - 1)) == 0))
571 return true1;
572
573 if (flags & PIN_MAPPABLE(1ULL << (3)) && !i915_vma_is_map_and_fenceable(vma))
574 return true1;
575
576 if (flags & PIN_OFFSET_BIAS(1ULL << (6)) &&
577 vma->node.start < (flags & PIN_OFFSET_MASK-(1ULL << (12))))
578 return true1;
579
580 if (flags & PIN_OFFSET_FIXED(1ULL << (7)) &&
581 vma->node.start != (flags & PIN_OFFSET_MASK-(1ULL << (12))))
582 return true1;
583
584 return false0;
585}
586
587void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
588{
589 bool_Bool mappable, fenceable;
590
591 GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0);
592 GEM_BUG_ON(!vma->fence_size)((void)0);
593
594 fenceable = (vma->node.size >= vma->fence_size &&
595 IS_ALIGNED(vma->node.start, vma->fence_alignment)(((vma->node.start) & ((vma->fence_alignment) - 1))
== 0)
);
596
597 mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end;
598
599 if (mappable && fenceable)
600 set_bit(I915_VMA_CAN_FENCE_BIT15, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
601 else
602 clear_bit(I915_VMA_CAN_FENCE_BIT15, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
603}
604
605bool_Bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
606{
607 struct drm_mm_node *node = &vma->node;
608 struct drm_mm_node *other;
609
610 /*
611 * On some machines we have to be careful when putting differing types
612 * of snoopable memory together to avoid the prefetcher crossing memory
613 * domains and dying. During vm initialisation, we decide whether or not
614 * these constraints apply and set the drm_mm.color_adjust
615 * appropriately.
616 */
617 if (!i915_vm_has_cache_coloring(vma->vm))
618 return true1;
619
620 /* Only valid to be called on an already inserted vma */
621 GEM_BUG_ON(!drm_mm_node_allocated(node))((void)0);
622 GEM_BUG_ON(list_empty(&node->node_list))((void)0);
623
624 other = list_prev_entry(node, node_list)({ const __typeof( ((typeof(*(node)) *)0)->node_list ) *__mptr
= (((node)->node_list.prev)); (typeof(*(node)) *)( (char *
)__mptr - __builtin_offsetof(typeof(*(node)), node_list) );})
;
625 if (i915_node_color_differs(other, color) &&
626 !drm_mm_hole_follows(other))
627 return false0;
628
629 other = list_next_entry(node, node_list)({ const __typeof( ((typeof(*(node)) *)0)->node_list ) *__mptr
= (((node)->node_list.next)); (typeof(*(node)) *)( (char *
)__mptr - __builtin_offsetof(typeof(*(node)), node_list) );})
;
630 if (i915_node_color_differs(other, color) &&
631 !drm_mm_hole_follows(node))
632 return false0;
633
634 return true1;
635}
636
637/**
638 * i915_vma_insert - finds a slot for the vma in its address space
639 * @vma: the vma
640 * @size: requested size in bytes (can be larger than the VMA)
641 * @alignment: required alignment
642 * @flags: mask of PIN_* flags to use
643 *
644 * First we try to allocate some free space that meets the requirements for
645 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
646 * preferrably the oldest idle entry to make room for the new VMA.
647 *
648 * Returns:
649 * 0 on success, negative error code otherwise.
650 */
651static int
652i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
653{
654 unsigned long color;
655 u64 start, end;
656 int ret;
657
658 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))((void)0);
659 GEM_BUG_ON(drm_mm_node_allocated(&vma->node))((void)0);
660
661 size = max(size, vma->size)(((size)>(vma->size))?(size):(vma->size));
662 alignment = max(alignment, vma->display_alignment)(((alignment)>(vma->display_alignment))?(alignment):(vma
->display_alignment))
;
663 if (flags & PIN_MAPPABLE(1ULL << (3))) {
664 size = max_t(typeof(size), size, vma->fence_size)({ typeof(size) __max_a = (size); typeof(size) __max_b = (vma
->fence_size); __max_a > __max_b ? __max_a : __max_b; }
)
;
665 alignment = max_t(typeof(alignment),({ typeof(alignment) __max_a = (alignment); typeof(alignment)
__max_b = (vma->fence_alignment); __max_a > __max_b ? __max_a
: __max_b; })
666 alignment, vma->fence_alignment)({ typeof(alignment) __max_a = (alignment); typeof(alignment)
__max_b = (vma->fence_alignment); __max_a > __max_b ? __max_a
: __max_b; })
;
667 }
668
669 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE))((void)0);
670 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT))((void)0);
671 GEM_BUG_ON(!is_power_of_2(alignment))((void)0);
672
673 start = flags & PIN_OFFSET_BIAS(1ULL << (6)) ? flags & PIN_OFFSET_MASK-(1ULL << (12)) : 0;
674 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE))((void)0);
675
676 end = vma->vm->total;
677 if (flags & PIN_MAPPABLE(1ULL << (3)))
678 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end)({ u64 __min_a = (end); u64 __min_b = (i915_vm_to_ggtt(vma->
vm)->mappable_end); __min_a < __min_b ? __min_a : __min_b
; })
;
679 if (flags & PIN_ZONE_4G(1ULL << (4)))
680 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE)({ u64 __min_a = (end); u64 __min_b = ((1ULL << 32) - (
1ULL << (12))); __min_a < __min_b ? __min_a : __min_b
; })
;
681 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE))((void)0);
682
683 /* If binding the object/GGTT view requires more space than the entire
684 * aperture has, reject it early before evicting everything in a vain
685 * attempt to find space.
686 */
687 if (size > end) {
688 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",__drm_dbg(DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n"
, size, flags & (1ULL << (3)) ? "mappable" : "total"
, end)
689 size, flags & PIN_MAPPABLE ? "mappable" : "total",__drm_dbg(DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n"
, size, flags & (1ULL << (3)) ? "mappable" : "total"
, end)
690 end)__drm_dbg(DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n"
, size, flags & (1ULL << (3)) ? "mappable" : "total"
, end)
;
691 return -ENOSPC28;
692 }
693
694 color = 0;
695 if (vma->obj && i915_vm_has_cache_coloring(vma->vm))
696 color = vma->obj->cache_level;
697
698 if (flags & PIN_OFFSET_FIXED(1ULL << (7))) {
699 u64 offset = flags & PIN_OFFSET_MASK-(1ULL << (12));
700 if (!IS_ALIGNED(offset, alignment)(((offset) & ((alignment) - 1)) == 0) ||
701 range_overflows(offset, size, end)({ typeof(offset) start__ = (offset); typeof(size) size__ = (
size); typeof(end) max__ = (end); (void)(&start__ == &
size__); (void)(&start__ == &max__); start__ >= max__
|| size__ > max__ - start__; })
)
702 return -EINVAL22;
703
704 ret = i915_gem_gtt_reserve(vma->vm, &vma->node,
705 size, offset, color,
706 flags);
707 if (ret)
708 return ret;
709 } else {
710 /*
711 * We only support huge gtt pages through the 48b PPGTT,
712 * however we also don't want to force any alignment for
713 * objects which need to be tightly packed into the low 32bits.
714 *
715 * Note that we assume that GGTT are limited to 4GiB for the
716 * forseeable future. See also i915_ggtt_offset().
717 */
718 if (upper_32_bits(end - 1)((u32)(((end - 1) >> 16) >> 16)) &&
719 vma->page_sizes.sg > I915_GTT_PAGE_SIZE(1ULL << (12))) {
720 /*
721 * We can't mix 64K and 4K PTEs in the same page-table
722 * (2M block), and so to avoid the ugliness and
723 * complexity of coloring we opt for just aligning 64K
724 * objects to 2M.
725 */
726 u64 page_alignment =
727 rounddown_pow_of_two(vma->page_sizes.sg |
728 I915_GTT_PAGE_SIZE_2M(1ULL << (21)));
729
730 /*
731 * Check we don't expand for the limited Global GTT
732 * (mappable aperture is even more precious!). This
733 * also checks that we exclude the aliasing-ppgtt.
734 */
735 GEM_BUG_ON(i915_vma_is_ggtt(vma))((void)0);
736
737 alignment = max(alignment, page_alignment)(((alignment)>(page_alignment))?(alignment):(page_alignment
))
;
738
739 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K(1ULL << (16)))
740 size = round_up(size, I915_GTT_PAGE_SIZE_2M)((((size) + (((1ULL << (21))) - 1)) / ((1ULL << (
21)))) * ((1ULL << (21))))
;
741 }
742
743 ret = i915_gem_gtt_insert(vma->vm, &vma->node,
744 size, alignment, color,
745 start, end, flags);
746 if (ret)
747 return ret;
748
749 GEM_BUG_ON(vma->node.start < start)((void)0);
750 GEM_BUG_ON(vma->node.start + vma->node.size > end)((void)0);
751 }
752 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0);
753 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color))((void)0);
754
755 list_add_tail(&vma->vm_link, &vma->vm->bound_list);
756
757 return 0;
758}
759
760static void
761i915_vma_detach(struct i915_vma *vma)
762{
763 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0);
764 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))((void)0);
765
766 /*
767 * And finally now the object is completely decoupled from this
768 * vma, we can drop its hold on the backing storage and allow
769 * it to be reaped by the shrinker.
770 */
771 list_del(&vma->vm_link);
772}
773
774static bool_Bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
775{
776 unsigned int bound;
777 bool_Bool pinned = true1;
778
779 bound = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*(
&vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer
(); __tmp; })
;
780 do {
781 if (unlikely(flags & ~bound)__builtin_expect(!!(flags & ~bound), 0))
782 return false0;
783
784 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))__builtin_expect(!!(bound & (0x200 | ((int)(1UL << (
13))))), 0)
)
785 return false0;
786
787 if (!(bound & I915_VMA_PIN_MASK0x3ff))
788 goto unpinned;
789
790 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0)((void)0);
791 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
792
793 return true1;
794
795unpinned:
796 /*
797 * If pin_count==0, but we are bound, check under the lock to avoid
798 * racing with a concurrent i915_vma_unbind().
799 */
800 mutex_lock(&vma->vm->mutex)rw_enter_write(&vma->vm->mutex);
801 do {
802 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))__builtin_expect(!!(bound & (0x200 | ((int)(1UL << (
13))))), 0)
) {
803 pinned = false0;
804 break;
805 }
806
807 if (unlikely(flags & ~bound)__builtin_expect(!!(flags & ~bound), 0)) {
808 pinned = false0;
809 break;
810 }
811 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
812 mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex);
813
814 return pinned;
815}
816
817static int vma_get_pages(struct i915_vma *vma)
818{
819 int err = 0;
820
821 if (atomic_add_unless(&vma->pages_count, 1, 0))
822 return 0;
823
824 /* Allocations ahoy! */
825 if (mutex_lock_interruptible(&vma->pages_mutex))
826 return -EINTR4;
827
828 if (!atomic_read(&vma->pages_count)({ typeof(*(&vma->pages_count)) __tmp = *(volatile typeof
(*(&vma->pages_count)) *)&(*(&vma->pages_count
)); membar_datadep_consumer(); __tmp; })
) {
829 if (vma->obj) {
830 err = i915_gem_object_pin_pages(vma->obj);
831 if (err)
832 goto unlock;
833 }
834
835 err = vma->ops->set_pages(vma);
836 if (err) {
837 if (vma->obj)
838 i915_gem_object_unpin_pages(vma->obj);
839 goto unlock;
840 }
841 }
842 atomic_inc(&vma->pages_count)__sync_fetch_and_add(&vma->pages_count, 1);
843
844unlock:
845 mutex_unlock(&vma->pages_mutex)rw_exit_write(&vma->pages_mutex);
846
847 return err;
848}
849
850static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
851{
852 /* We allocate under vma_get_pages, so beware the shrinker */
853 mutex_lock_nested(&vma->pages_mutex, SINGLE_DEPTH_NESTING)rw_enter_write(&vma->pages_mutex);
854 GEM_BUG_ON(atomic_read(&vma->pages_count) < count)((void)0);
855 if (atomic_sub_return(count, &vma->pages_count)__sync_sub_and_fetch(&vma->pages_count, count) == 0) {
856 vma->ops->clear_pages(vma);
857 GEM_BUG_ON(vma->pages)((void)0);
858 if (vma->obj)
859 i915_gem_object_unpin_pages(vma->obj);
860 }
861 mutex_unlock(&vma->pages_mutex)rw_exit_write(&vma->pages_mutex);
862}
863
864static void vma_put_pages(struct i915_vma *vma)
865{
866 if (atomic_add_unless(&vma->pages_count, -1, 1))
867 return;
868
869 __vma_put_pages(vma, 1);
870}
871
872static void vma_unbind_pages(struct i915_vma *vma)
873{
874 unsigned int count;
875
876 lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0);
877
878 /* The upper portion of pages_count is the number of bindings */
879 count = atomic_read(&vma->pages_count)({ typeof(*(&vma->pages_count)) __tmp = *(volatile typeof
(*(&vma->pages_count)) *)&(*(&vma->pages_count
)); membar_datadep_consumer(); __tmp; })
;
880 count >>= I915_VMA_PAGES_BIAS24;
881 GEM_BUG_ON(!count)((void)0);
882
883 __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS24);
884}
885
886int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
887 u64 size, u64 alignment, u64 flags)
888{
889 struct i915_vma_work *work = NULL((void *)0);
890 intel_wakeref_t wakeref = 0;
891 unsigned int bound;
892 int err;
893
894#ifdef CONFIG_PROVE_LOCKING
895 if (debug_locks && lockdep_is_held(&vma->vm->i915->drm.struct_mutex))
896 WARN_ON(!ww)({ int __ret = !!((!ww)); if (__ret) printf("%s", "WARN_ON(" "!ww"
")"); __builtin_expect(!!(__ret), 0); })
;
897#endif
898
899 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND)extern char _ctassert[(!((1ULL << (10)) != ((int)(1UL <<
(10))))) ? 1 : -1 ] __attribute__((__unused__))
;
900 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND)extern char _ctassert[(!((1ULL << (11)) != ((int)(1UL <<
(11))))) ? 1 : -1 ] __attribute__((__unused__))
;
901
902 GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)))((void)0);
903
904 /* First try and grab the pin without rebinding the vma */
905 if (try_qad_pin(vma, flags & I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11))))))
906 return 0;
907
908 err = vma_get_pages(vma);
909 if (err)
910 return err;
911
912 if (flags & PIN_GLOBAL(1ULL << (10)))
913 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
914
915 if (flags & vma->vm->bind_async_flags) {
916 work = i915_vma_work();
917 if (!work) {
918 err = -ENOMEM12;
919 goto err_rpm;
920 }
921
922 work->vm = i915_vm_get(vma->vm);
923
924 /* Allocate enough page directories to used PTE */
925 if (vma->vm->allocate_va_range) {
926 err = i915_vm_alloc_pt_stash(vma->vm,
927 &work->stash,
928 vma->size);
929 if (err)
930 goto err_fence;
931
932 err = i915_vm_pin_pt_stash(vma->vm,
933 &work->stash);
934 if (err)
935 goto err_fence;
936 }
937 }
938
939 /*
940 * Differentiate between user/kernel vma inside the aliasing-ppgtt.
941 *
942 * We conflate the Global GTT with the user's vma when using the
943 * aliasing-ppgtt, but it is still vitally important to try and
944 * keep the use cases distinct. For example, userptr objects are
945 * not allowed inside the Global GTT as that will cause lock
946 * inversions when we have to evict them the mmu_notifier callbacks -
947 * but they are allowed to be part of the user ppGTT which can never
948 * be mapped. As such we try to give the distinct users of the same
949 * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
950 * and i915_ppgtt separate].
951 *
952 * NB this may cause us to mask real lock inversions -- while the
953 * code is safe today, lockdep may not be able to spot future
954 * transgressions.
955 */
956 err = mutex_lock_interruptible_nested(&vma->vm->mutex,mutex_lock_interruptible(&vma->vm->mutex)
957 !(flags & PIN_GLOBAL))mutex_lock_interruptible(&vma->vm->mutex);
958 if (err)
959 goto err_fence;
960
961 /* No more allocations allowed now we hold vm->mutex */
962
963 if (unlikely(i915_vma_is_closed(vma))__builtin_expect(!!(i915_vma_is_closed(vma)), 0)) {
964 err = -ENOENT2;
965 goto err_unlock;
966 }
967
968 bound = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*(
&vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer
(); __tmp; })
;
969 if (unlikely(bound & I915_VMA_ERROR)__builtin_expect(!!(bound & ((int)(1UL << (13)))), 0
)
) {
970 err = -ENOMEM12;
971 goto err_unlock;
972 }
973
974 if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))__builtin_expect(!!(!((bound + 1) & 0x3ff)), 0)) {
975 err = -EAGAIN35; /* pins are meant to be fairly temporary */
976 goto err_unlock;
977 }
978
979 if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))__builtin_expect(!!(!(flags & ~bound & (((int)(1UL <<
(10))) | ((int)(1UL << (11)))))), 0)
) {
980 __i915_vma_pin(vma);
981 goto err_unlock;
982 }
983
984 err = i915_active_acquire(&vma->active);
985 if (err)
986 goto err_unlock;
987
988 if (!(bound & I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) {
989 err = i915_vma_insert(vma, size, alignment, flags);
990 if (err)
991 goto err_active;
992
993 if (i915_is_ggtt(vma->vm)((vma->vm)->is_ggtt))
994 __i915_vma_set_map_and_fenceable(vma);
995 }
996
997 GEM_BUG_ON(!vma->pages)((void)0);
998 err = i915_vma_bind(vma,
999 vma->obj ? vma->obj->cache_level : 0,
1000 flags, work);
1001 if (err)
1002 goto err_remove;
1003
1004 /* There should only be at most 2 active bindings (user, global) */
1005 GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound)((void)0);
1006 atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count)__sync_fetch_and_add(&vma->pages_count, ((1UL <<
(24)) | 1))
;
1007 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1008
1009 __i915_vma_pin(vma);
1010 GEM_BUG_ON(!i915_vma_is_pinned(vma))((void)0);
1011 GEM_BUG_ON(!i915_vma_is_bound(vma, flags))((void)0);
1012 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags))((void)0);
1013
1014err_remove:
1015 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) {
1016 i915_vma_detach(vma);
1017 drm_mm_remove_node(&vma->node);
1018 }
1019err_active:
1020 i915_active_release(&vma->active);
1021err_unlock:
1022 mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex);
1023err_fence:
1024 if (work)
1025 dma_fence_work_commit_imm(&work->base);
1026err_rpm:
1027 if (wakeref)
1028 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
1029 vma_put_pages(vma);
1030 return err;
1031}
1032
1033static void flush_idle_contexts(struct intel_gt *gt)
1034{
1035 struct intel_engine_cs *engine;
1036 enum intel_engine_id id;
1037
1038 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else
1039 intel_engine_flush_barriers(engine);
1040
1041 intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT(0x7fffffff));
1042}
1043
1044int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1045 u32 align, unsigned int flags)
1046{
1047 struct i915_address_space *vm = vma->vm;
1048 int err;
1049
1050 GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0);
1051
1052 do {
1053 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL(1ULL << (10)));
1054 if (err != -ENOSPC28) {
1055 if (!err) {
1056 err = i915_vma_wait_for_bind(vma);
1057 if (err)
1058 i915_vma_unpin(vma);
1059 }
1060 return err;
1061 }
1062
1063 /* Unlike i915_vma_pin, we don't take no for an answer! */
1064 flush_idle_contexts(vm->gt);
1065 if (mutex_lock_interruptible(&vm->mutex) == 0) {
1066 i915_gem_evict_vm(vm);
1067 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
1068 }
1069 } while (1);
1070}
1071
1072static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1073{
1074 /*
1075 * We defer actually closing, unbinding and destroying the VMA until
1076 * the next idle point, or if the object is freed in the meantime. By
1077 * postponing the unbind, we allow for it to be resurrected by the
1078 * client, avoiding the work required to rebind the VMA. This is
1079 * advantageous for DRI, where the client/server pass objects
1080 * between themselves, temporarily opening a local VMA to the
1081 * object, and then closing it again. The same object is then reused
1082 * on the next frame (or two, depending on the depth of the swap queue)
1083 * causing us to rebind the VMA once more. This ends up being a lot
1084 * of wasted work for the steady state.
1085 */
1086 GEM_BUG_ON(i915_vma_is_closed(vma))((void)0);
1087 list_add(&vma->closed_link, &gt->closed_vma);
1088}
1089
1090void i915_vma_close(struct i915_vma *vma)
1091{
1092 struct intel_gt *gt = vma->vm->gt;
1093 unsigned long flags;
1094
1095 if (i915_vma_is_ggtt(vma))
1096 return;
1097
1098 GEM_BUG_ON(!atomic_read(&vma->open_count))((void)0);
1099 if (atomic_dec_and_lock_irqsave(&vma->open_count,atomic_dec_and_lock(&vma->open_count, &gt->closed_lock
)
1100 &gt->closed_lock,atomic_dec_and_lock(&vma->open_count, &gt->closed_lock
)
1101 flags)atomic_dec_and_lock(&vma->open_count, &gt->closed_lock
)
) {
1102 __vma_close(vma, gt);
1103 spin_unlock_irqrestore(&gt->closed_lock, flags)do { (void)(flags); mtx_leave(&gt->closed_lock); } while
(0)
;
1104 }
1105}
1106
1107static void __i915_vma_remove_closed(struct i915_vma *vma)
1108{
1109 struct intel_gt *gt = vma->vm->gt;
1110
1111 spin_lock_irq(&gt->closed_lock)mtx_enter(&gt->closed_lock);
1112 list_del_init(&vma->closed_link);
1113 spin_unlock_irq(&gt->closed_lock)mtx_leave(&gt->closed_lock);
1114}
1115
1116void i915_vma_reopen(struct i915_vma *vma)
1117{
1118 if (i915_vma_is_closed(vma))
1119 __i915_vma_remove_closed(vma);
1120}
1121
1122void i915_vma_release(struct kref *ref)
1123{
1124 struct i915_vma *vma = container_of(ref, typeof(*vma), ref)({ const __typeof( ((typeof(*vma) *)0)->ref ) *__mptr = (ref
); (typeof(*vma) *)( (char *)__mptr - __builtin_offsetof(typeof
(*vma), ref) );})
;
1125
1126 if (drm_mm_node_allocated(&vma->node)) {
1127 mutex_lock(&vma->vm->mutex)rw_enter_write(&vma->vm->mutex);
1128 atomic_and(~I915_VMA_PIN_MASK, &vma->flags)__sync_fetch_and_and(&vma->flags, ~0x3ff);
1129 WARN_ON(__i915_vma_unbind(vma))({ int __ret = !!((__i915_vma_unbind(vma))); if (__ret) printf
("%s", "WARN_ON(" "__i915_vma_unbind(vma)" ")"); __builtin_expect
(!!(__ret), 0); })
;
1130 mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex);
1131 GEM_BUG_ON(drm_mm_node_allocated(&vma->node))((void)0);
1132 }
1133 GEM_BUG_ON(i915_vma_is_active(vma))((void)0);
1134
1135 if (vma->obj) {
1136 struct drm_i915_gem_object *obj = vma->obj;
1137
1138 spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
1139 list_del(&vma->obj_link);
1140 if (!RB_EMPTY_NODE(&vma->obj_node)((&vma->obj_node)->__entry.rbe_parent == &vma->
obj_node)
)
1141 rb_erase(&vma->obj_node, &obj->vma.tree)linux_root_RB_REMOVE((struct linux_root *)(&obj->vma.tree
), (&vma->obj_node))
;
1142 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
1143 }
1144
1145 __i915_vma_remove_closed(vma);
1146 i915_vm_put(vma->vm);
1147
1148 i915_active_fini(&vma->active);
1149 i915_vma_free(vma);
1150}
1151
1152void i915_vma_parked(struct intel_gt *gt)
1153{
1154 struct i915_vma *vma, *next;
1155 DRM_LIST_HEAD(closed)struct list_head closed = { &(closed), &(closed) };
1156
1157 spin_lock_irq(&gt->closed_lock)mtx_enter(&gt->closed_lock);
1158 list_for_each_entry_safe(vma, next, &gt->closed_vma, closed_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = ((&gt->closed_vma)->next); (__typeof(*
vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link
) );}), next = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = (vma->closed_link.next); (__typeof(*vma) *)( (
char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link
) );}); &vma->closed_link != (&gt->closed_vma);
vma = next, next = ({ const __typeof( ((__typeof(*next) *)0)
->closed_link ) *__mptr = (next->closed_link.next); (__typeof
(*next) *)( (char *)__mptr - __builtin_offsetof(__typeof(*next
), closed_link) );}))
{
1159 struct drm_i915_gem_object *obj = vma->obj;
1160 struct i915_address_space *vm = vma->vm;
1161
1162 /* XXX All to avoid keeping a reference on i915_vma itself */
1163
1164 if (!kref_get_unless_zero(&obj->base.refcount))
1165 continue;
1166
1167 if (!i915_vm_tryopen(vm)) {
1168 i915_gem_object_put(obj);
1169 continue;
1170 }
1171
1172 list_move(&vma->closed_link, &closed);
1173 }
1174 spin_unlock_irq(&gt->closed_lock)mtx_leave(&gt->closed_lock);
1175
1176 /* As the GT is held idle, no vma can be reopened as we destroy them */
1177 list_for_each_entry_safe(vma, next, &closed, closed_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = ((&closed)->next); (__typeof(*vma) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*vma), closed_link) )
;}), next = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = (vma->closed_link.next); (__typeof(*vma) *)( (
char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link
) );}); &vma->closed_link != (&closed); vma = next
, next = ({ const __typeof( ((__typeof(*next) *)0)->closed_link
) *__mptr = (next->closed_link.next); (__typeof(*next) *)
( (char *)__mptr - __builtin_offsetof(__typeof(*next), closed_link
) );}))
{
1178 struct drm_i915_gem_object *obj = vma->obj;
1179 struct i915_address_space *vm = vma->vm;
1180
1181 INIT_LIST_HEAD(&vma->closed_link);
1182 __i915_vma_put(vma);
1183
1184 i915_gem_object_put(obj);
1185 i915_vm_close(vm);
1186 }
1187}
1188
1189static void __i915_vma_iounmap(struct i915_vma *vma)
1190{
1191 GEM_BUG_ON(i915_vma_is_pinned(vma))((void)0);
1192
1193 if (vma->iomap == NULL((void *)0))
1194 return;
1195
1196#ifdef __linux__
1197 io_mapping_unmap(vma->iomap);
1198#else
1199 struct drm_i915_privateinteldrm_softc *dev_priv = vma->vm->i915;
1200 agp_unmap_subregion(dev_priv->agph, vma->bsh, vma->node.size);
1201#endif
1202 vma->iomap = NULL((void *)0);
1203}
1204
1205void i915_vma_revoke_mmap(struct i915_vma *vma)
1206{
1207 struct drm_vma_offset_node *node;
1208 u64 vma_offset;
1209
1210 if (!i915_vma_has_userfault(vma))
1211 return;
1212
1213 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma))((void)0);
1214 GEM_BUG_ON(!vma->obj->userfault_count)((void)0);
1215
1216 node = &vma->mmo->vma_node;
Value stored to 'node' is never read
1217 vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT12;
1218#ifdef __linux__
1219 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
1220 drm_vma_node_offset_addr(node) + vma_offset,
1221 vma->size,
1222 1);
1223#else
1224 struct drm_i915_privateinteldrm_softc *dev_priv = vma->obj->base.dev->dev_private;
1225 struct vm_page *pg;
1226
1227 for (pg = &dev_priv->pgs[atop(vma->node.start)((vma->node.start) >> 12)];
1228 pg != &dev_priv->pgs[atop(vma->node.start + vma->size)((vma->node.start + vma->size) >> 12)];
1229 pg++)
1230 pmap_page_protect(pg, PROT_NONE0x00);
1231#endif
1232
1233 i915_vma_unset_userfault(vma);
1234 if (!--vma->obj->userfault_count)
1235 list_del(&vma->obj->userfault_link);
1236}
1237
1238static int
1239__i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1240{
1241 return __i915_request_await_exclusive(rq, &vma->active);
1242}
1243
1244int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1245{
1246 int err;
1247
1248 GEM_BUG_ON(!i915_vma_is_pinned(vma))((void)0);
1249
1250 /* Wait for the vma to be bound before we start! */
1251 err = __i915_request_await_bind(rq, vma);
1252 if (err)
1253 return err;
1254
1255 return i915_active_add_request(&vma->active, rq);
1256}
1257
1258int i915_vma_move_to_active(struct i915_vma *vma,
1259 struct i915_request *rq,
1260 unsigned int flags)
1261{
1262 struct drm_i915_gem_object *obj = vma->obj;
1263 int err;
1264
1265 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0)
;
1266
1267 err = __i915_vma_move_to_active(vma, rq);
1268 if (unlikely(err)__builtin_expect(!!(err), 0))
1269 return err;
1270
1271 if (flags & EXEC_OBJECT_WRITE(1<<2)) {
1272 struct intel_frontbuffer *front;
1273
1274 front = __intel_frontbuffer_get(obj);
1275 if (unlikely(front)__builtin_expect(!!(front), 0)) {
1276 if (intel_frontbuffer_invalidate(front, ORIGIN_CS))
1277 i915_active_add_request(&front->write, rq);
1278 intel_frontbuffer_put(front);
1279 }
1280
1281 dma_resv_add_excl_fence(vma->resv, &rq->fence);
1282 obj->write_domain = I915_GEM_DOMAIN_RENDER0x00000002;
1283 obj->read_domains = 0;
1284 } else {
1285 err = dma_resv_reserve_shared(vma->resv, 1);
1286 if (unlikely(err)__builtin_expect(!!(err), 0))
1287 return err;
1288
1289 dma_resv_add_shared_fence(vma->resv, &rq->fence);
1290 obj->write_domain = 0;
1291 }
1292
1293 if (flags & EXEC_OBJECT_NEEDS_FENCE(1<<0) && vma->fence)
1294 i915_active_add_request(&vma->fence->active, rq);
1295
1296 obj->read_domains |= I915_GEM_GPU_DOMAINS(0x00000002 | 0x00000004 | 0x00000008 | 0x00000010 | 0x00000020
)
;
1297 obj->mm.dirty = true1;
1298
1299 GEM_BUG_ON(!i915_vma_is_active(vma))((void)0);
1300 return 0;
1301}
1302
1303void __i915_vma_evict(struct i915_vma *vma)
1304{
1305 GEM_BUG_ON(i915_vma_is_pinned(vma))((void)0);
1306
1307 if (i915_vma_is_map_and_fenceable(vma)) {
1308 /* Force a pagefault for domain tracking on next user access */
1309 i915_vma_revoke_mmap(vma);
1310
1311 /*
1312 * Check that we have flushed all writes through the GGTT
1313 * before the unbind, other due to non-strict nature of those
1314 * indirect writes they may end up referencing the GGTT PTE
1315 * after the unbind.
1316 *
1317 * Note that we may be concurrently poking at the GGTT_WRITE
1318 * bit from set-domain, as we mark all GGTT vma associated
1319 * with an object. We know this is for another vma, as we
1320 * are currently unbinding this one -- so if this vma will be
1321 * reused, it will be refaulted and have its dirty bit set
1322 * before the next write.
1323 */
1324 i915_vma_flush_writes(vma);
1325
1326 /* release the fence reg _after_ flushing */
1327 i915_vma_revoke_fence(vma);
1328
1329 __i915_vma_iounmap(vma);
1330 clear_bit(I915_VMA_CAN_FENCE_BIT15, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
1331 }
1332 GEM_BUG_ON(vma->fence)((void)0);
1333 GEM_BUG_ON(i915_vma_has_userfault(vma))((void)0);
1334
1335 if (likely(atomic_read(&vma->vm->open))__builtin_expect(!!(({ typeof(*(&vma->vm->open)) __tmp
= *(volatile typeof(*(&vma->vm->open)) *)&(*(&
vma->vm->open)); membar_datadep_consumer(); __tmp; })),
1)
) {
1336 trace_i915_vma_unbind(vma);
1337 vma->ops->unbind_vma(vma->vm, vma);
1338 }
1339 atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),__sync_fetch_and_and(&vma->flags, ~((((int)(1UL <<
(10))) | ((int)(1UL << (11)))) | ((int)(1UL << (
13))) | ((int)(1UL << (17)))))
1340 &vma->flags)__sync_fetch_and_and(&vma->flags, ~((((int)(1UL <<
(10))) | ((int)(1UL << (11)))) | ((int)(1UL << (
13))) | ((int)(1UL << (17)))))
;
1341
1342 i915_vma_detach(vma);
1343 vma_unbind_pages(vma);
1344}
1345
1346int __i915_vma_unbind(struct i915_vma *vma)
1347{
1348 int ret;
1349
1350 lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0);
1351
1352 if (!drm_mm_node_allocated(&vma->node))
1353 return 0;
1354
1355 if (i915_vma_is_pinned(vma)) {
1356 vma_print_allocator(vma, "is pinned");
1357 return -EAGAIN35;
1358 }
1359
1360 /*
1361 * After confirming that no one else is pinning this vma, wait for
1362 * any laggards who may have crept in during the wait (through
1363 * a residual pin skipping the vm->mutex) to complete.
1364 */
1365 ret = i915_vma_sync(vma);
1366 if (ret)
1367 return ret;
1368
1369 GEM_BUG_ON(i915_vma_is_active(vma))((void)0);
1370 __i915_vma_evict(vma);
1371
1372 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
1373 return 0;
1374}
1375
1376int i915_vma_unbind(struct i915_vma *vma)
1377{
1378 struct i915_address_space *vm = vma->vm;
1379 intel_wakeref_t wakeref = 0;
1380 int err;
1381
1382 /* Optimistic wait before taking the mutex */
1383 err = i915_vma_sync(vma);
1384 if (err)
1385 return err;
1386
1387 if (!drm_mm_node_allocated(&vma->node))
1388 return 0;
1389
1390 if (i915_vma_is_pinned(vma)) {
1391 vma_print_allocator(vma, "is pinned");
1392 return -EAGAIN35;
1393 }
1394
1395 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10)))))
1396 /* XXX not always required: nop_clear_range */
1397 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
1398
1399 err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref)mutex_lock_interruptible(&vma->vm->mutex);
1400 if (err)
1401 goto out_rpm;
1402
1403 err = __i915_vma_unbind(vma);
1404 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
1405
1406out_rpm:
1407 if (wakeref)
1408 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
1409 return err;
1410}
1411
1412struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
1413{
1414 i915_gem_object_make_unshrinkable(vma->obj);
1415 return vma;
1416}
1417
1418void i915_vma_make_shrinkable(struct i915_vma *vma)
1419{
1420 i915_gem_object_make_shrinkable(vma->obj);
1421}
1422
1423void i915_vma_make_purgeable(struct i915_vma *vma)
1424{
1425 i915_gem_object_make_purgeable(vma->obj);
1426}
1427
1428#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
1429#include "selftests/i915_vma.c"
1430#endif
1431
1432static void i915_global_vma_shrink(void)
1433{
1434#ifdef notyet
1435 kmem_cache_shrink(global.slab_vmas);
1436#endif
1437}
1438
1439static void i915_global_vma_exit(void)
1440{
1441#ifdef __linux__
1442 kmem_cache_destroy(global.slab_vmas);
1443#else
1444 pool_destroy(&global.slab_vmas);
1445#endif
1446}
1447
1448static struct i915_global_vma global = { {
1449 .shrink = i915_global_vma_shrink,
1450 .exit = i915_global_vma_exit,
1451} };
1452
1453int __init i915_global_vma_init(void)
1454{
1455#ifdef __linux__
1456 global.slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
1457 if (!global.slab_vmas)
1458 return -ENOMEM12;
1459#else
1460 pool_init(&global.slab_vmas, sizeof(struct i915_vma),
1461 CACHELINESIZE64, IPL_NONE0x0, 0, "drmvma", NULL((void *)0));
1462#endif
1463
1464 i915_global_register(&global.base);
1465 return 0;
1466}