Bug Summary

File:dev/pci/drm/i915/i915_vma.c
Warning:line 1841, column 2
Value stored to 'node' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name i915_vma.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/i915_vma.c
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/sched/mm.h>
26#include <linux/dma-fence-array.h>
27#include <drm/drm_gem.h>
28
29#include "display/intel_frontbuffer.h"
30#include "gem/i915_gem_lmem.h"
31#include "gem/i915_gem_tiling.h"
32#include "gt/intel_engine.h"
33#include "gt/intel_engine_heartbeat.h"
34#include "gt/intel_gt.h"
35#include "gt/intel_gt_requests.h"
36
37#include "i915_drv.h"
38#include "i915_gem_evict.h"
39#include "i915_sw_fence_work.h"
40#include "i915_trace.h"
41#include "i915_vma.h"
42#include "i915_vma_resource.h"
43
44#include <dev/pci/agpvar.h>
45
46static inline void assert_vma_held_evict(const struct i915_vma *vma)
47{
48 /*
49 * We may be forced to unbind when the vm is dead, to clean it up.
50 * This is the only exception to the requirement of the object lock
51 * being held.
52 */
53 if (kref_read(&vma->vm->ref))
54 assert_object_held_shared(vma->obj);
55}
56
57static struct pool slab_vmas;
58
59static struct i915_vma *i915_vma_alloc(void)
60{
61#ifdef __linux__
62 return kmem_cache_zalloc(slab_vmas, GFP_KERNEL(0x0001 | 0x0004));
63#else
64 return pool_get(&slab_vmas, PR_WAITOK0x0001 | PR_ZERO0x0008);
65#endif
66}
67
68static void i915_vma_free(struct i915_vma *vma)
69{
70#ifdef __linux__
71 return kmem_cache_free(slab_vmas, vma);
72#else
73 pool_put(&slab_vmas, vma);
74#endif
75}
76
77#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM)0 && IS_ENABLED(CONFIG_DRM_DEBUG_MM)0
78
79#include <linux/stackdepot.h>
80
81static void vma_print_allocator(struct i915_vma *vma, const char *reason)
82{
83 char buf[512];
84
85 if (!vma->node.stack) {
86 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",___drm_dbg(((void *)0), DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: unknown owner\n"
, vma->node.start, vma->node.size, reason)
87 vma->node.start, vma->node.size, reason)___drm_dbg(((void *)0), DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: unknown owner\n"
, vma->node.start, vma->node.size, reason)
;
88 return;
89 }
90
91 stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0);
92 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",___drm_dbg(((void *)0), DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: inserted at %s\n"
, vma->node.start, vma->node.size, reason, buf)
93 vma->node.start, vma->node.size, reason, buf)___drm_dbg(((void *)0), DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: inserted at %s\n"
, vma->node.start, vma->node.size, reason, buf)
;
94}
95
96#else
97
98static void vma_print_allocator(struct i915_vma *vma, const char *reason)
99{
100}
101
102#endif
103
104static inline struct i915_vma *active_to_vma(struct i915_active *ref)
105{
106 return container_of(ref, typeof(struct i915_vma), active)({ const __typeof( ((typeof(struct i915_vma) *)0)->active )
*__mptr = (ref); (typeof(struct i915_vma) *)( (char *)__mptr
- __builtin_offsetof(typeof(struct i915_vma), active) );})
;
107}
108
109static int __i915_vma_active(struct i915_active *ref)
110{
111 return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT2;
112}
113
114static void __i915_vma_retire(struct i915_active *ref)
115{
116 i915_vma_put(active_to_vma(ref));
117}
118
119static struct i915_vma *
120vma_create(struct drm_i915_gem_object *obj,
121 struct i915_address_space *vm,
122 const struct i915_gtt_view *view)
123{
124 struct i915_vma *pos = ERR_PTR(-E2BIG7);
125 struct i915_vma *vma;
126 struct rb_node *rb, **p;
127 int err;
128
129 /* The aliasing_ppgtt should never be used directly! */
130 GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm)((void)0);
131
132 vma = i915_vma_alloc();
133 if (vma == NULL((void *)0))
134 return ERR_PTR(-ENOMEM12);
135
136 vma->ops = &vm->vma_ops;
137 vma->obj = obj;
138 vma->size = obj->base.size;
139 vma->display_alignment = I915_GTT_MIN_ALIGNMENT(1ULL << (12));
140
141 i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0)do { static struct lock_class_key __mkey; static struct lock_class_key
__wkey; __i915_active_init(&vma->active, __i915_vma_active
, __i915_vma_retire, 0, &__mkey, &__wkey); } while (0
)
;
142
143#ifdef notyet
144 /* Declare ourselves safe for use inside shrinkers */
145 if (IS_ENABLED(CONFIG_LOCKDEP)0) {
146 fs_reclaim_acquire(GFP_KERNEL(0x0001 | 0x0004));
147 might_lock(&vma->active.mutex);
148 fs_reclaim_release(GFP_KERNEL(0x0001 | 0x0004));
149 }
150#endif
151
152 INIT_LIST_HEAD(&vma->closed_link);
153 INIT_LIST_HEAD(&vma->obj_link);
154 RB_CLEAR_NODE(&vma->obj_node)(((&vma->obj_node))->__entry.rbe_parent = (&vma
->obj_node))
;
155
156 if (view && view->type != I915_GTT_VIEW_NORMAL) {
157 vma->gtt_view = *view;
158 if (view->type == I915_GTT_VIEW_PARTIAL) {
159 GEM_BUG_ON(range_overflows_t(u64,((void)0)
160 view->partial.offset,((void)0)
161 view->partial.size,((void)0)
162 obj->base.size >> PAGE_SHIFT))((void)0);
163 vma->size = view->partial.size;
164 vma->size <<= PAGE_SHIFT12;
165 GEM_BUG_ON(vma->size > obj->base.size)((void)0);
166 } else if (view->type == I915_GTT_VIEW_ROTATED) {
167 vma->size = intel_rotation_info_size(&view->rotated);
168 vma->size <<= PAGE_SHIFT12;
169 } else if (view->type == I915_GTT_VIEW_REMAPPED) {
170 vma->size = intel_remapped_info_size(&view->remapped);
171 vma->size <<= PAGE_SHIFT12;
172 }
173 }
174
175 if (unlikely(vma->size > vm->total)__builtin_expect(!!(vma->size > vm->total), 0))
176 goto err_vma;
177
178 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE))((void)0);
179
180 err = mutex_lock_interruptible(&vm->mutex);
181 if (err) {
182 pos = ERR_PTR(err);
183 goto err_vma;
184 }
185
186 vma->vm = vm;
187 list_add_tail(&vma->vm_link, &vm->unbound_list);
188
189 spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
190 if (i915_is_ggtt(vm)((vm)->is_ggtt)) {
191 if (unlikely(overflows_type(vma->size, u32))__builtin_expect(!!((sizeof(vma->size) > sizeof(u32) &&
(vma->size) >> (8 * sizeof(u32)))), 0)
)
192 goto err_unlock;
193
194 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size,
195 i915_gem_object_get_tiling(obj),
196 i915_gem_object_get_stride(obj));
197 if (unlikely(vma->fence_size < vma->size || /* overflow */__builtin_expect(!!(vma->fence_size < vma->size || vma
->fence_size > vm->total), 0)
198 vma->fence_size > vm->total)__builtin_expect(!!(vma->fence_size < vma->size || vma
->fence_size > vm->total), 0)
)
199 goto err_unlock;
200
201 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT))((void)0);
202
203 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size,
204 i915_gem_object_get_tiling(obj),
205 i915_gem_object_get_stride(obj));
206 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment))((void)0);
207
208 __set_bit(I915_VMA_GGTT_BIT13, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
209 }
210
211 rb = NULL((void *)0);
212 p = &obj->vma.tree.rb_node;
213 while (*p) {
214 long cmp;
215
216 rb = *p;
217 pos = rb_entry(rb, struct i915_vma, obj_node)({ const __typeof( ((struct i915_vma *)0)->obj_node ) *__mptr
= (rb); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof
(struct i915_vma, obj_node) );})
;
218
219 /*
220 * If the view already exists in the tree, another thread
221 * already created a matching vma, so return the older instance
222 * and dispose of ours.
223 */
224 cmp = i915_vma_compare(pos, vm, view);
225 if (cmp < 0)
226 p = &rb->rb_right__entry.rbe_right;
227 else if (cmp > 0)
228 p = &rb->rb_left__entry.rbe_left;
229 else
230 goto err_unlock;
231 }
232 rb_link_node(&vma->obj_node, rb, p);
233 rb_insert_color(&vma->obj_node, &obj->vma.tree)linux_root_RB_INSERT_COLOR((struct linux_root *)(&obj->
vma.tree), (&vma->obj_node))
;
234
235 if (i915_vma_is_ggtt(vma))
236 /*
237 * We put the GGTT vma at the start of the vma-list, followed
238 * by the ppGGTT vma. This allows us to break early when
239 * iterating over only the GGTT vma for an object, see
240 * for_each_ggtt_vma()
241 */
242 list_add(&vma->obj_link, &obj->vma.list);
243 else
244 list_add_tail(&vma->obj_link, &obj->vma.list);
245
246 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
247 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
248
249 return vma;
250
251err_unlock:
252 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
253 list_del_init(&vma->vm_link);
254 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
255err_vma:
256 i915_vma_free(vma);
257 return pos;
258}
259
260static struct i915_vma *
261i915_vma_lookup(struct drm_i915_gem_object *obj,
262 struct i915_address_space *vm,
263 const struct i915_gtt_view *view)
264{
265 struct rb_node *rb;
266
267 rb = obj->vma.tree.rb_node;
268 while (rb) {
269 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node)({ const __typeof( ((struct i915_vma *)0)->obj_node ) *__mptr
= (rb); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof
(struct i915_vma, obj_node) );})
;
270 long cmp;
271
272 cmp = i915_vma_compare(vma, vm, view);
273 if (cmp == 0)
274 return vma;
275
276 if (cmp < 0)
277 rb = rb->rb_right__entry.rbe_right;
278 else
279 rb = rb->rb_left__entry.rbe_left;
280 }
281
282 return NULL((void *)0);
283}
284
285/**
286 * i915_vma_instance - return the singleton instance of the VMA
287 * @obj: parent &struct drm_i915_gem_object to be mapped
288 * @vm: address space in which the mapping is located
289 * @view: additional mapping requirements
290 *
291 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
292 * the same @view characteristics. If a match is not found, one is created.
293 * Once created, the VMA is kept until either the object is freed, or the
294 * address space is closed.
295 *
296 * Returns the vma, or an error pointer.
297 */
298struct i915_vma *
299i915_vma_instance(struct drm_i915_gem_object *obj,
300 struct i915_address_space *vm,
301 const struct i915_gtt_view *view)
302{
303 struct i915_vma *vma;
304
305 GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm))((void)0);
306 GEM_BUG_ON(!kref_read(&vm->ref))((void)0);
307
308 spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
309 vma = i915_vma_lookup(obj, vm, view);
310 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
311
312 /* vma_create() will resolve the race if another creates the vma */
313 if (unlikely(!vma)__builtin_expect(!!(!vma), 0))
314 vma = vma_create(obj, vm, view);
315
316 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view))((void)0);
317 return vma;
318}
319
320struct i915_vma_work {
321 struct dma_fence_work base;
322 struct i915_address_space *vm;
323 struct i915_vm_pt_stash stash;
324 struct i915_vma_resource *vma_res;
325 struct drm_i915_gem_object *obj;
326 struct i915_sw_dma_fence_cb cb;
327 enum i915_cache_level cache_level;
328 unsigned int flags;
329};
330
331static void __vma_bind(struct dma_fence_work *work)
332{
333 struct i915_vma_work *vw = container_of(work, typeof(*vw), base)({ const __typeof( ((typeof(*vw) *)0)->base ) *__mptr = (work
); (typeof(*vw) *)( (char *)__mptr - __builtin_offsetof(typeof
(*vw), base) );})
;
334 struct i915_vma_resource *vma_res = vw->vma_res;
335
336 /*
337 * We are about the bind the object, which must mean we have already
338 * signaled the work to potentially clear/move the pages underneath. If
339 * something went wrong at that stage then the object should have
340 * unknown_state set, in which case we need to skip the bind.
341 */
342 if (i915_gem_object_has_unknown_state(vw->obj))
343 return;
344
345 vma_res->ops->bind_vma(vma_res->vm, &vw->stash,
346 vma_res, vw->cache_level, vw->flags);
347}
348
349static void __vma_release(struct dma_fence_work *work)
350{
351 struct i915_vma_work *vw = container_of(work, typeof(*vw), base)({ const __typeof( ((typeof(*vw) *)0)->base ) *__mptr = (work
); (typeof(*vw) *)( (char *)__mptr - __builtin_offsetof(typeof
(*vw), base) );})
;
352
353 if (vw->obj)
354 i915_gem_object_put(vw->obj);
355
356 i915_vm_free_pt_stash(vw->vm, &vw->stash);
357 if (vw->vma_res)
358 i915_vma_resource_put(vw->vma_res);
359}
360
361static const struct dma_fence_work_ops bind_ops = {
362 .name = "bind",
363 .work = __vma_bind,
364 .release = __vma_release,
365};
366
367struct i915_vma_work *i915_vma_work(void)
368{
369 struct i915_vma_work *vw;
370
371 vw = kzalloc(sizeof(*vw), GFP_KERNEL(0x0001 | 0x0004));
372 if (!vw)
373 return NULL((void *)0);
374
375 dma_fence_work_init(&vw->base, &bind_ops);
376 vw->base.dma.error = -EAGAIN35; /* disable the worker by default */
377
378 return vw;
379}
380
381int i915_vma_wait_for_bind(struct i915_vma *vma)
382{
383 int err = 0;
384
385 if (rcu_access_pointer(vma->active.excl.fence)(vma->active.excl.fence)) {
386 struct dma_fence *fence;
387
388 rcu_read_lock();
389 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence);
390 rcu_read_unlock();
391 if (fence) {
392 err = dma_fence_wait(fence, true1);
393 dma_fence_put(fence);
394 }
395 }
396
397 return err;
398}
399
400#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)0
401static int i915_vma_verify_bind_complete(struct i915_vma *vma)0
402{
403 struct dma_fence *fence = i915_active_fence_get(&vma->active.excl);
404 int err;
405
406 if (!fence)
407 return 0;
408
409 if (dma_fence_is_signaled(fence))
410 err = fence->error;
411 else
412 err = -EBUSY16;
413
414 dma_fence_put(fence);
415
416 return err;
417}
418#else
419#define i915_vma_verify_bind_complete(_vma)0 0
420#endif
421
422I915_SELFTEST_EXPORTstatic void
423i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
424 struct i915_vma *vma)
425{
426 struct drm_i915_gem_object *obj = vma->obj;
427
428 i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes,
429 obj->mm.rsgt, i915_gem_object_is_readonly(obj),
430 i915_gem_object_is_lmem(obj), obj->mm.region,
431 vma->ops, vma->private, vma->node.start,
432 vma->node.size, vma->size);
433}
434
435/**
436 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
437 * @vma: VMA to map
438 * @cache_level: mapping cache level
439 * @flags: flags like global or local mapping
440 * @work: preallocated worker for allocating and binding the PTE
441 * @vma_res: pointer to a preallocated vma resource. The resource is either
442 * consumed or freed.
443 *
444 * DMA addresses are taken from the scatter-gather table of this object (or of
445 * this VMA in case of non-default GGTT views) and PTE entries set up.
446 * Note that DMA addresses are also the only part of the SG table we care about.
447 */
448int i915_vma_bind(struct i915_vma *vma,
449 enum i915_cache_level cache_level,
450 u32 flags,
451 struct i915_vma_work *work,
452 struct i915_vma_resource *vma_res)
453{
454 u32 bind_flags;
455 u32 vma_flags;
456 int ret;
457
458 lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0);
459 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0);
460 GEM_BUG_ON(vma->size > vma->node.size)((void)0);
461
462 if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,({ ((void)0); 0; })
463 vma->node.size,({ ((void)0); 0; })
464 vma->vm->total))({ ((void)0); 0; })) {
465 i915_vma_resource_free(vma_res);
466 return -ENODEV19;
467 }
468
469 if (GEM_DEBUG_WARN_ON(!flags)({ ((void)0); 0; })) {
470 i915_vma_resource_free(vma_res);
471 return -EINVAL22;
472 }
473
474 bind_flags = flags;
475 bind_flags &= I915_VMA_GLOBAL_BIND((int)(1UL << (10))) | I915_VMA_LOCAL_BIND((int)(1UL << (11)));
476
477 vma_flags = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*(
&vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer
(); __tmp; })
;
478 vma_flags &= I915_VMA_GLOBAL_BIND((int)(1UL << (10))) | I915_VMA_LOCAL_BIND((int)(1UL << (11)));
479
480 bind_flags &= ~vma_flags;
481 if (bind_flags == 0) {
482 i915_vma_resource_free(vma_res);
483 return 0;
484 }
485
486 GEM_BUG_ON(!atomic_read(&vma->pages_count))((void)0);
487
488 /* Wait for or await async unbinds touching our range */
489 if (work && bind_flags & vma->vm->bind_async_flags)
490 ret = i915_vma_resource_bind_dep_await(vma->vm,
491 &work->base.chain,
492 vma->node.start,
493 vma->node.size,
494 true1,
495 GFP_NOWAIT0x0002 |
496 __GFP_RETRY_MAYFAIL0 |
497 __GFP_NOWARN0);
498 else
499 ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start,
500 vma->node.size, true1);
501 if (ret) {
502 i915_vma_resource_free(vma_res);
503 return ret;
504 }
505
506 if (vma->resource || !vma_res) {
507 /* Rebinding with an additional I915_VMA_*_BIND */
508 GEM_WARN_ON(!vma_flags)({ __builtin_expect(!!(!!(!vma_flags)), 0); });
509 i915_vma_resource_free(vma_res);
510 } else {
511 i915_vma_resource_init_from_vma(vma_res, vma);
512 vma->resource = vma_res;
513 }
514 trace_i915_vma_bind(vma, bind_flags);
515 if (work && bind_flags & vma->vm->bind_async_flags) {
516 struct dma_fence *prev;
517
518 work->vma_res = i915_vma_resource_get(vma->resource);
519 work->cache_level = cache_level;
520 work->flags = bind_flags;
521
522 /*
523 * Note we only want to chain up to the migration fence on
524 * the pages (not the object itself). As we don't track that,
525 * yet, we have to use the exclusive fence instead.
526 *
527 * Also note that we do not want to track the async vma as
528 * part of the obj->resv->excl_fence as it only affects
529 * execution and not content or object's backing store lifetime.
530 */
531 prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
532 if (prev) {
533 __i915_sw_fence_await_dma_fence(&work->base.chain,
534 prev,
535 &work->cb);
536 dma_fence_put(prev);
537 }
538
539 work->base.dma.error = 0; /* enable the queue_work() */
540 work->obj = i915_gem_object_get(vma->obj);
541 } else {
542 ret = i915_gem_object_wait_moving_fence(vma->obj, true1);
543 if (ret) {
544 i915_vma_resource_free(vma->resource);
545 vma->resource = NULL((void *)0);
546
547 return ret;
548 }
549 vma->ops->bind_vma(vma->vm, NULL((void *)0), vma->resource, cache_level,
550 bind_flags);
551 }
552
553 atomic_or(bind_flags, &vma->flags)x86_atomic_setbits_u32(&vma->flags, bind_flags);
554 return 0;
555}
556
557void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
558{
559 void __iomem *ptr;
560 int err;
561
562 if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY)({ static int __warned; int __ret = !!(vma->obj->flags &
(1UL << (6))); if (__ret && !__warned) { printf
("WARNING %s failed at %s:%d\n", "vma->obj->flags & (1UL << (6))"
, "/usr/src/sys/dev/pci/drm/i915/i915_vma.c", 562); __warned =
1; } __builtin_expect(!!(__ret), 0); })
)
563 return IOMEM_ERR_PTR(-EINVAL22);
564
565 GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0);
566 GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))((void)0);
567 GEM_BUG_ON(i915_vma_verify_bind_complete(vma))((void)0);
568
569 ptr = READ_ONCE(vma->iomap)({ typeof(vma->iomap) __tmp = *(volatile typeof(vma->iomap
) *)&(vma->iomap); membar_datadep_consumer(); __tmp; }
)
;
570 if (ptr == NULL((void *)0)) {
571 /*
572 * TODO: consider just using i915_gem_object_pin_map() for lmem
573 * instead, which already supports mapping non-contiguous chunks
574 * of pages, that way we can also drop the
575 * I915_BO_ALLOC_CONTIGUOUS when allocating the object.
576 */
577 if (i915_gem_object_is_lmem(vma->obj)) {
578 ptr = i915_gem_object_lmem_io_map(vma->obj, 0,
579 vma->obj->base.size);
580 } else if (i915_vma_is_map_and_fenceable(vma)) {
581#ifdef __linux__
582 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
583 vma->node.start,
584 vma->node.size);
585#else
586 {
587 struct drm_i915_privateinteldrm_softc *dev_priv = vma->vm->i915;
588 err = agp_map_subregion(dev_priv->agph, vma->node.start,
589 vma->node.size, &vma->bsh);
590 if (err) {
591 err = -err;
592 goto err;
593 }
594 ptr = bus_space_vaddr(dev_priv->bst, vma->bsh)((dev_priv->bst)->vaddr((vma->bsh)));
595 }
596#endif
597 } else {
598 ptr = (void __iomem *)
599 i915_gem_object_pin_map(vma->obj, I915_MAP_WC);
600 if (IS_ERR(ptr)) {
601 err = PTR_ERR(ptr);
602 goto err;
603 }
604 ptr = page_pack_bits(ptr, 1)({ unsigned long __bits = (1); ((void)0); ((typeof(ptr))((unsigned
long)(ptr) | __bits)); })
;
605 }
606
607 if (ptr == NULL((void *)0)) {
608 err = -ENOMEM12;
609 goto err;
610 }
611
612 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))__builtin_expect(!!(__sync_val_compare_and_swap(&vma->
iomap, ((void *)0), ptr)), 0)
) {
613 if (page_unmask_bits(ptr)((unsigned long)(ptr) & ((1UL << (12)) - 1)))
614 __i915_gem_object_release_map(vma->obj);
615#ifdef __linux__
616 else
617 io_mapping_unmap(ptr);
618#endif
619 ptr = vma->iomap;
620 }
621 }
622
623 __i915_vma_pin(vma);
624
625 err = i915_vma_pin_fence(vma);
626 if (err)
627 goto err_unpin;
628
629 i915_vma_set_ggtt_write(vma);
630
631 /* NB Access through the GTT requires the device to be awake. */
632 return page_mask_bits(ptr)({ unsigned long __v = (unsigned long)(ptr); (typeof(ptr))(__v
& -(1UL << (12))); })
;
633
634err_unpin:
635 __i915_vma_unpin(vma);
636err:
637 return IOMEM_ERR_PTR(err);
638}
639
640void i915_vma_flush_writes(struct i915_vma *vma)
641{
642 if (i915_vma_unset_ggtt_write(vma))
643 intel_gt_flush_ggtt_writes(vma->vm->gt);
644}
645
646void i915_vma_unpin_iomap(struct i915_vma *vma)
647{
648 GEM_BUG_ON(vma->iomap == NULL)((void)0);
649
650 /* XXX We keep the mapping until __i915_vma_unbind()/evict() */
651
652 i915_vma_flush_writes(vma);
653
654 i915_vma_unpin_fence(vma);
655 i915_vma_unpin(vma);
656}
657
658void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
659{
660 struct i915_vma *vma;
661 struct drm_i915_gem_object *obj;
662
663 vma = fetch_and_zero(p_vma)({ typeof(*p_vma) __T = *(p_vma); *(p_vma) = (typeof(*p_vma))
0; __T; })
;
664 if (!vma)
665 return;
666
667 obj = vma->obj;
668 GEM_BUG_ON(!obj)((void)0);
669
670 i915_vma_unpin(vma);
671
672 if (flags & I915_VMA_RELEASE_MAP(1UL << (0)))
673 i915_gem_object_unpin_map(obj);
674
675 i915_gem_object_put(obj);
676}
677
678bool_Bool i915_vma_misplaced(const struct i915_vma *vma,
679 u64 size, u64 alignment, u64 flags)
680{
681 if (!drm_mm_node_allocated(&vma->node))
682 return false0;
683
684 if (test_bit(I915_VMA_ERROR_BIT12, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags)))
685 return true1;
686
687 if (vma->node.size < size)
688 return true1;
689
690 GEM_BUG_ON(alignment && !is_power_of_2(alignment))((void)0);
691 if (alignment && !IS_ALIGNED(vma->node.start, alignment)(((vma->node.start) & ((alignment) - 1)) == 0))
692 return true1;
693
694 if (flags & PIN_MAPPABLE(1ULL << (3)) && !i915_vma_is_map_and_fenceable(vma))
695 return true1;
696
697 if (flags & PIN_OFFSET_BIAS(1ULL << (6)) &&
698 vma->node.start < (flags & PIN_OFFSET_MASK-(1ULL << (12))))
699 return true1;
700
701 if (flags & PIN_OFFSET_FIXED(1ULL << (7)) &&
702 vma->node.start != (flags & PIN_OFFSET_MASK-(1ULL << (12))))
703 return true1;
704
705 return false0;
706}
707
708void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
709{
710 bool_Bool mappable, fenceable;
711
712 GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0);
713 GEM_BUG_ON(!vma->fence_size)((void)0);
714
715 fenceable = (vma->node.size >= vma->fence_size &&
716 IS_ALIGNED(vma->node.start, vma->fence_alignment)(((vma->node.start) & ((vma->fence_alignment) - 1))
== 0)
);
717
718 mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end;
719
720 if (mappable && fenceable)
721 set_bit(I915_VMA_CAN_FENCE_BIT14, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
722 else
723 clear_bit(I915_VMA_CAN_FENCE_BIT14, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
724}
725
726bool_Bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
727{
728 struct drm_mm_node *node = &vma->node;
729 struct drm_mm_node *other;
730
731 /*
732 * On some machines we have to be careful when putting differing types
733 * of snoopable memory together to avoid the prefetcher crossing memory
734 * domains and dying. During vm initialisation, we decide whether or not
735 * these constraints apply and set the drm_mm.color_adjust
736 * appropriately.
737 */
738 if (!i915_vm_has_cache_coloring(vma->vm))
739 return true1;
740
741 /* Only valid to be called on an already inserted vma */
742 GEM_BUG_ON(!drm_mm_node_allocated(node))((void)0);
743 GEM_BUG_ON(list_empty(&node->node_list))((void)0);
744
745 other = list_prev_entry(node, node_list)({ const __typeof( ((typeof(*(node)) *)0)->node_list ) *__mptr
= (((node)->node_list.prev)); (typeof(*(node)) *)( (char *
)__mptr - __builtin_offsetof(typeof(*(node)), node_list) );})
;
746 if (i915_node_color_differs(other, color) &&
747 !drm_mm_hole_follows(other))
748 return false0;
749
750 other = list_next_entry(node, node_list)({ const __typeof( ((typeof(*(node)) *)0)->node_list ) *__mptr
= (((node)->node_list.next)); (typeof(*(node)) *)( (char *
)__mptr - __builtin_offsetof(typeof(*(node)), node_list) );})
;
751 if (i915_node_color_differs(other, color) &&
752 !drm_mm_hole_follows(node))
753 return false0;
754
755 return true1;
756}
757
758/**
759 * i915_vma_insert - finds a slot for the vma in its address space
760 * @vma: the vma
761 * @size: requested size in bytes (can be larger than the VMA)
762 * @alignment: required alignment
763 * @flags: mask of PIN_* flags to use
764 *
765 * First we try to allocate some free space that meets the requirements for
766 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
767 * preferrably the oldest idle entry to make room for the new VMA.
768 *
769 * Returns:
770 * 0 on success, negative error code otherwise.
771 */
772static int
773i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
774 u64 size, u64 alignment, u64 flags)
775{
776 unsigned long color;
777 u64 start, end;
778 int ret;
779
780 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))((void)0);
781 GEM_BUG_ON(drm_mm_node_allocated(&vma->node))((void)0);
782
783 size = max(size, vma->size)(((size)>(vma->size))?(size):(vma->size));
784 alignment = max(alignment, vma->display_alignment)(((alignment)>(vma->display_alignment))?(alignment):(vma
->display_alignment))
;
785 if (flags & PIN_MAPPABLE(1ULL << (3))) {
786 size = max_t(typeof(size), size, vma->fence_size)({ typeof(size) __max_a = (size); typeof(size) __max_b = (vma
->fence_size); __max_a > __max_b ? __max_a : __max_b; }
)
;
787 alignment = max_t(typeof(alignment),({ typeof(alignment) __max_a = (alignment); typeof(alignment)
__max_b = (vma->fence_alignment); __max_a > __max_b ? __max_a
: __max_b; })
788 alignment, vma->fence_alignment)({ typeof(alignment) __max_a = (alignment); typeof(alignment)
__max_b = (vma->fence_alignment); __max_a > __max_b ? __max_a
: __max_b; })
;
789 }
790
791 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE))((void)0);
792 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT))((void)0);
793 GEM_BUG_ON(!is_power_of_2(alignment))((void)0);
794
795 start = flags & PIN_OFFSET_BIAS(1ULL << (6)) ? flags & PIN_OFFSET_MASK-(1ULL << (12)) : 0;
796 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE))((void)0);
797
798 end = vma->vm->total;
799 if (flags & PIN_MAPPABLE(1ULL << (3)))
800 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end)({ u64 __min_a = (end); u64 __min_b = (i915_vm_to_ggtt(vma->
vm)->mappable_end); __min_a < __min_b ? __min_a : __min_b
; })
;
801 if (flags & PIN_ZONE_4G(1ULL << (4)))
802 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE)({ u64 __min_a = (end); u64 __min_b = ((1ULL << 32) - (
1ULL << (12))); __min_a < __min_b ? __min_a : __min_b
; })
;
803 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE))((void)0);
804
805 alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj))(((alignment)>(i915_vm_obj_min_alignment(vma->vm, vma->
obj)))?(alignment):(i915_vm_obj_min_alignment(vma->vm, vma
->obj)))
;
806 /*
807 * for compact-pt we round up the reservation to prevent
808 * any smaller pages being used within the same PDE
809 */
810 if (NEEDS_COMPACT_PT(vma->vm->i915)((&(vma->vm->i915)->__info)->needs_compact_pt
)
)
811 size = round_up(size, alignment)((((size) + ((alignment) - 1)) / (alignment)) * (alignment));
812
813 /* If binding the object/GGTT view requires more space than the entire
814 * aperture has, reject it early before evicting everything in a vain
815 * attempt to find space.
816 */
817 if (size > end) {
818 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",___drm_dbg(((void *)0), DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n"
, size, flags & (1ULL << (3)) ? "mappable" : "total"
, end)
819 size, flags & PIN_MAPPABLE ? "mappable" : "total",___drm_dbg(((void *)0), DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n"
, size, flags & (1ULL << (3)) ? "mappable" : "total"
, end)
820 end)___drm_dbg(((void *)0), DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n"
, size, flags & (1ULL << (3)) ? "mappable" : "total"
, end)
;
821 return -ENOSPC28;
822 }
823
824 color = 0;
825
826 if (i915_vm_has_cache_coloring(vma->vm))
827 color = vma->obj->cache_level;
828
829 if (flags & PIN_OFFSET_FIXED(1ULL << (7))) {
830 u64 offset = flags & PIN_OFFSET_MASK-(1ULL << (12));
831 if (!IS_ALIGNED(offset, alignment)(((offset) & ((alignment) - 1)) == 0) ||
832 range_overflows(offset, size, end)({ typeof(offset) start__ = (offset); typeof(size) size__ = (
size); typeof(end) max__ = (end); (void)(&start__ == &
size__); (void)(&start__ == &max__); start__ >= max__
|| size__ > max__ - start__; })
)
833 return -EINVAL22;
834
835 ret = i915_gem_gtt_reserve(vma->vm, ww, &vma->node,
836 size, offset, color,
837 flags);
838 if (ret)
839 return ret;
840 } else {
841 /*
842 * We only support huge gtt pages through the 48b PPGTT,
843 * however we also don't want to force any alignment for
844 * objects which need to be tightly packed into the low 32bits.
845 *
846 * Note that we assume that GGTT are limited to 4GiB for the
847 * forseeable future. See also i915_ggtt_offset().
848 */
849 if (upper_32_bits(end - 1)((u32)(((end - 1) >> 16) >> 16)) &&
850 vma->page_sizes.sg > I915_GTT_PAGE_SIZE(1ULL << (12))) {
851 /*
852 * We can't mix 64K and 4K PTEs in the same page-table
853 * (2M block), and so to avoid the ugliness and
854 * complexity of coloring we opt for just aligning 64K
855 * objects to 2M.
856 */
857 u64 page_alignment =
858 rounddown_pow_of_two(vma->page_sizes.sg |
859 I915_GTT_PAGE_SIZE_2M(1ULL << (21)));
860
861 /*
862 * Check we don't expand for the limited Global GTT
863 * (mappable aperture is even more precious!). This
864 * also checks that we exclude the aliasing-ppgtt.
865 */
866 GEM_BUG_ON(i915_vma_is_ggtt(vma))((void)0);
867
868 alignment = max(alignment, page_alignment)(((alignment)>(page_alignment))?(alignment):(page_alignment
))
;
869
870 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K(1ULL << (16)))
871 size = round_up(size, I915_GTT_PAGE_SIZE_2M)((((size) + (((1ULL << (21))) - 1)) / ((1ULL << (
21)))) * ((1ULL << (21))))
;
872 }
873
874 ret = i915_gem_gtt_insert(vma->vm, ww, &vma->node,
875 size, alignment, color,
876 start, end, flags);
877 if (ret)
878 return ret;
879
880 GEM_BUG_ON(vma->node.start < start)((void)0);
881 GEM_BUG_ON(vma->node.start + vma->node.size > end)((void)0);
882 }
883 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0);
884 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color))((void)0);
885
886 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
887
888 return 0;
889}
890
891static void
892i915_vma_detach(struct i915_vma *vma)
893{
894 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0);
895 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))((void)0);
896
897 /*
898 * And finally now the object is completely decoupled from this
899 * vma, we can drop its hold on the backing storage and allow
900 * it to be reaped by the shrinker.
901 */
902 list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
903}
904
905static bool_Bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
906{
907 unsigned int bound;
908
909 bound = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*(
&vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer
(); __tmp; })
;
910
911 if (flags & PIN_VALIDATE(1ULL << (8))) {
912 flags &= I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11))));
913
914 return (flags & bound) == flags;
915 }
916
917 /* with the lock mandatory for unbind, we don't race here */
918 flags &= I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11))));
919 do {
920 if (unlikely(flags & ~bound)__builtin_expect(!!(flags & ~bound), 0))
921 return false0;
922
923 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))__builtin_expect(!!(bound & (0x200 | ((int)(1UL << (
12))))), 0)
)
924 return false0;
925
926 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0)((void)0);
927 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
928
929 return true1;
930}
931
932static struct scatterlist *
933rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
934 unsigned int width, unsigned int height,
935 unsigned int src_stride, unsigned int dst_stride,
936 struct sg_table *st, struct scatterlist *sg)
937{
938 unsigned int column, row;
939 unsigned int src_idx;
940
941 for (column = 0; column < width; column++) {
942 unsigned int left;
943
944 src_idx = src_stride * (height - 1) + column + offset;
945 for (row = 0; row < height; row++) {
946 st->nents++;
947 /*
948 * We don't need the pages, but need to initialize
949 * the entries so the sg list can be happily traversed.
950 * The only thing we need are DMA addresses.
951 */
952 sg_set_page(sg, NULL((void *)0), I915_GTT_PAGE_SIZE(1ULL << (12)), 0);
953 sg_dma_address(sg)((sg)->dma_address) =
954 i915_gem_object_get_dma_address(obj, src_idx);
955 sg_dma_len(sg)((sg)->length) = I915_GTT_PAGE_SIZE(1ULL << (12));
956 sg = sg_next(sg);
957 src_idx -= src_stride;
958 }
959
960 left = (dst_stride - height) * I915_GTT_PAGE_SIZE(1ULL << (12));
961
962 if (!left)
963 continue;
964
965 st->nents++;
966
967 /*
968 * The DE ignores the PTEs for the padding tiles, the sg entry
969 * here is just a conenience to indicate how many padding PTEs
970 * to insert at this spot.
971 */
972 sg_set_page(sg, NULL((void *)0), left, 0);
973 sg_dma_address(sg)((sg)->dma_address) = 0;
974 sg_dma_len(sg)((sg)->length) = left;
975 sg = sg_next(sg);
976 }
977
978 return sg;
979}
980
981static noinline__attribute__((__noinline__)) struct sg_table *
982intel_rotate_pages(struct intel_rotation_info *rot_info,
983 struct drm_i915_gem_object *obj)
984{
985 unsigned int size = intel_rotation_info_size(rot_info);
986 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
987 struct sg_table *st;
988 struct scatterlist *sg;
989 int ret = -ENOMEM12;
990 int i;
991
992 /* Allocate target SG list. */
993 st = kmalloc(sizeof(*st), GFP_KERNEL(0x0001 | 0x0004));
994 if (!st)
995 goto err_st_alloc;
996
997 ret = sg_alloc_table(st, size, GFP_KERNEL(0x0001 | 0x0004));
998 if (ret)
999 goto err_sg_alloc;
1000
1001 st->nents = 0;
1002 sg = st->sgl;
1003
1004 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane)(sizeof((rot_info->plane)) / sizeof((rot_info->plane)[0
]))
; i++)
1005 sg = rotate_pages(obj, rot_info->plane[i].offset,
1006 rot_info->plane[i].width, rot_info->plane[i].height,
1007 rot_info->plane[i].src_stride,
1008 rot_info->plane[i].dst_stride,
1009 st, sg);
1010
1011 return st;
1012
1013err_sg_alloc:
1014 kfree(st);
1015err_st_alloc:
1016
1017 drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n"
, obj->base.size, rot_info->plane[0].width, rot_info->
plane[0].height, size)
1018 obj->base.size, rot_info->plane[0].width,__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n"
, obj->base.size, rot_info->plane[0].width, rot_info->
plane[0].height, size)
1019 rot_info->plane[0].height, size)__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n"
, obj->base.size, rot_info->plane[0].width, rot_info->
plane[0].height, size)
;
1020
1021 return ERR_PTR(ret);
1022}
1023
1024static struct scatterlist *
1025add_padding_pages(unsigned int count,
1026 struct sg_table *st, struct scatterlist *sg)
1027{
1028 st->nents++;
1029
1030 /*
1031 * The DE ignores the PTEs for the padding tiles, the sg entry
1032 * here is just a convenience to indicate how many padding PTEs
1033 * to insert at this spot.
1034 */
1035 sg_set_page(sg, NULL((void *)0), count * I915_GTT_PAGE_SIZE(1ULL << (12)), 0);
1036 sg_dma_address(sg)((sg)->dma_address) = 0;
1037 sg_dma_len(sg)((sg)->length) = count * I915_GTT_PAGE_SIZE(1ULL << (12));
1038 sg = sg_next(sg);
1039
1040 return sg;
1041}
1042
1043static struct scatterlist *
1044remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj,
1045 unsigned int offset, unsigned int alignment_pad,
1046 unsigned int width, unsigned int height,
1047 unsigned int src_stride, unsigned int dst_stride,
1048 struct sg_table *st, struct scatterlist *sg,
1049 unsigned int *gtt_offset)
1050{
1051 unsigned int row;
1052
1053 if (!width || !height)
1054 return sg;
1055
1056 if (alignment_pad)
1057 sg = add_padding_pages(alignment_pad, st, sg);
1058
1059 for (row = 0; row < height; row++) {
1060 unsigned int left = width * I915_GTT_PAGE_SIZE(1ULL << (12));
1061
1062 while (left) {
1063 dma_addr_t addr;
1064 unsigned int length;
1065
1066 /*
1067 * We don't need the pages, but need to initialize
1068 * the entries so the sg list can be happily traversed.
1069 * The only thing we need are DMA addresses.
1070 */
1071
1072 addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1073
1074 length = min(left, length)(((left)<(length))?(left):(length));
1075
1076 st->nents++;
1077
1078 sg_set_page(sg, NULL((void *)0), length, 0);
1079 sg_dma_address(sg)((sg)->dma_address) = addr;
1080 sg_dma_len(sg)((sg)->length) = length;
1081 sg = sg_next(sg);
1082
1083 offset += length / I915_GTT_PAGE_SIZE(1ULL << (12));
1084 left -= length;
1085 }
1086
1087 offset += src_stride - width;
1088
1089 left = (dst_stride - width) * I915_GTT_PAGE_SIZE(1ULL << (12));
1090
1091 if (!left)
1092 continue;
1093
1094 sg = add_padding_pages(left >> PAGE_SHIFT12, st, sg);
1095 }
1096
1097 *gtt_offset += alignment_pad + dst_stride * height;
1098
1099 return sg;
1100}
1101
1102static struct scatterlist *
1103remap_contiguous_pages(struct drm_i915_gem_object *obj,
1104 unsigned int obj_offset,
1105 unsigned int count,
1106 struct sg_table *st, struct scatterlist *sg)
1107{
1108 struct scatterlist *iter;
1109 unsigned int offset;
1110
1111 iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset);
1112 GEM_BUG_ON(!iter)((void)0);
1113
1114 do {
1115 unsigned int len;
1116
1117 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),(((((iter)->length) - (offset << 12))<(count <<
12))?(((iter)->length) - (offset << 12)):(count <<
12))
1118 count << PAGE_SHIFT)(((((iter)->length) - (offset << 12))<(count <<
12))?(((iter)->length) - (offset << 12)):(count <<
12))
;
1119 sg_set_page(sg, NULL((void *)0), len, 0);
1120 sg_dma_address(sg)((sg)->dma_address) =
1121 sg_dma_address(iter)((iter)->dma_address) + (offset << PAGE_SHIFT12);
1122 sg_dma_len(sg)((sg)->length) = len;
1123
1124 st->nents++;
1125 count -= len >> PAGE_SHIFT12;
1126 if (count == 0)
1127 return sg;
1128
1129 sg = __sg_next(sg);
1130 iter = __sg_next(iter);
1131 offset = 0;
1132 } while (1);
1133}
1134
1135static struct scatterlist *
1136remap_linear_color_plane_pages(struct drm_i915_gem_object *obj,
1137 unsigned int obj_offset, unsigned int alignment_pad,
1138 unsigned int size,
1139 struct sg_table *st, struct scatterlist *sg,
1140 unsigned int *gtt_offset)
1141{
1142 if (!size)
1143 return sg;
1144
1145 if (alignment_pad)
1146 sg = add_padding_pages(alignment_pad, st, sg);
1147
1148 sg = remap_contiguous_pages(obj, obj_offset, size, st, sg);
1149 sg = sg_next(sg);
1150
1151 *gtt_offset += alignment_pad + size;
1152
1153 return sg;
1154}
1155
1156static struct scatterlist *
1157remap_color_plane_pages(const struct intel_remapped_info *rem_info,
1158 struct drm_i915_gem_object *obj,
1159 int color_plane,
1160 struct sg_table *st, struct scatterlist *sg,
1161 unsigned int *gtt_offset)
1162{
1163 unsigned int alignment_pad = 0;
1164
1165 if (rem_info->plane_alignment)
1166 alignment_pad = roundup2(*gtt_offset, rem_info->plane_alignment)(((*gtt_offset) + ((rem_info->plane_alignment) - 1)) &
(~((__typeof(*gtt_offset))(rem_info->plane_alignment) - 1
)))
- *gtt_offset;
1167
1168 if (rem_info->plane[color_plane].linear)
1169 sg = remap_linear_color_plane_pages(obj,
1170 rem_info->plane[color_plane].offset,
1171 alignment_pad,
1172 rem_info->plane[color_plane].size,
1173 st, sg,
1174 gtt_offset);
1175
1176 else
1177 sg = remap_tiled_color_plane_pages(obj,
1178 rem_info->plane[color_plane].offset,
1179 alignment_pad,
1180 rem_info->plane[color_plane].width,
1181 rem_info->plane[color_plane].height,
1182 rem_info->plane[color_plane].src_stride,
1183 rem_info->plane[color_plane].dst_stride,
1184 st, sg,
1185 gtt_offset);
1186
1187 return sg;
1188}
1189
1190static noinline__attribute__((__noinline__)) struct sg_table *
1191intel_remap_pages(struct intel_remapped_info *rem_info,
1192 struct drm_i915_gem_object *obj)
1193{
1194 unsigned int size = intel_remapped_info_size(rem_info);
1195 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
1196 struct sg_table *st;
1197 struct scatterlist *sg;
1198 unsigned int gtt_offset = 0;
1199 int ret = -ENOMEM12;
1200 int i;
1201
1202 /* Allocate target SG list. */
1203 st = kmalloc(sizeof(*st), GFP_KERNEL(0x0001 | 0x0004));
1204 if (!st)
1205 goto err_st_alloc;
1206
1207 ret = sg_alloc_table(st, size, GFP_KERNEL(0x0001 | 0x0004));
1208 if (ret)
1209 goto err_sg_alloc;
1210
1211 st->nents = 0;
1212 sg = st->sgl;
1213
1214 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane)(sizeof((rem_info->plane)) / sizeof((rem_info->plane)[0
]))
; i++)
1215 sg = remap_color_plane_pages(rem_info, obj, i, st, sg, &gtt_offset);
1216
1217 i915_sg_trim(st);
1218
1219 return st;
1220
1221err_sg_alloc:
1222 kfree(st);
1223err_st_alloc:
1224
1225 drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n"
, obj->base.size, rem_info->plane[0].width, rem_info->
plane[0].height, size)
1226 obj->base.size, rem_info->plane[0].width,__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n"
, obj->base.size, rem_info->plane[0].width, rem_info->
plane[0].height, size)
1227 rem_info->plane[0].height, size)__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n"
, obj->base.size, rem_info->plane[0].width, rem_info->
plane[0].height, size)
;
1228
1229 return ERR_PTR(ret);
1230}
1231
1232static noinline__attribute__((__noinline__)) struct sg_table *
1233intel_partial_pages(const struct i915_gtt_view *view,
1234 struct drm_i915_gem_object *obj)
1235{
1236 struct sg_table *st;
1237 struct scatterlist *sg;
1238 unsigned int count = view->partial.size;
1239 int ret = -ENOMEM12;
1240
1241 st = kmalloc(sizeof(*st), GFP_KERNEL(0x0001 | 0x0004));
1242 if (!st)
1243 goto err_st_alloc;
1244
1245 ret = sg_alloc_table(st, count, GFP_KERNEL(0x0001 | 0x0004));
1246 if (ret)
1247 goto err_sg_alloc;
1248
1249 st->nents = 0;
1250
1251 sg = remap_contiguous_pages(obj, view->partial.offset, count, st, st->sgl);
1252
1253 sg_mark_end(sg);
1254 i915_sg_trim(st); /* Drop any unused tail entries. */
1255
1256 return st;
1257
1258err_sg_alloc:
1259 kfree(st);
1260err_st_alloc:
1261 return ERR_PTR(ret);
1262}
1263
1264static int
1265__i915_vma_get_pages(struct i915_vma *vma)
1266{
1267 struct sg_table *pages;
1268
1269 /*
1270 * The vma->pages are only valid within the lifespan of the borrowed
1271 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1272 * must be the vma->pages. A simple rule is that vma->pages must only
1273 * be accessed when the obj->mm.pages are pinned.
1274 */
1275 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj))((void)0);
1276
1277 switch (vma->gtt_view.type) {
1278 default:
1279 GEM_BUG_ON(vma->gtt_view.type)((void)0);
1280 fallthroughdo {} while (0);
1281 case I915_GTT_VIEW_NORMAL:
1282 pages = vma->obj->mm.pages;
1283 break;
1284
1285 case I915_GTT_VIEW_ROTATED:
1286 pages =
1287 intel_rotate_pages(&vma->gtt_view.rotated, vma->obj);
1288 break;
1289
1290 case I915_GTT_VIEW_REMAPPED:
1291 pages =
1292 intel_remap_pages(&vma->gtt_view.remapped, vma->obj);
1293 break;
1294
1295 case I915_GTT_VIEW_PARTIAL:
1296 pages = intel_partial_pages(&vma->gtt_view, vma->obj);
1297 break;
1298 }
1299
1300 if (IS_ERR(pages)) {
1301 drm_err(&vma->vm->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to get pages for VMA view type %u (%ld)!\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , vma->
gtt_view.type, PTR_ERR(pages))
1302 "Failed to get pages for VMA view type %u (%ld)!\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to get pages for VMA view type %u (%ld)!\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , vma->
gtt_view.type, PTR_ERR(pages))
1303 vma->gtt_view.type, PTR_ERR(pages))printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to get pages for VMA view type %u (%ld)!\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , vma->
gtt_view.type, PTR_ERR(pages))
;
1304 return PTR_ERR(pages);
1305 }
1306
1307 vma->pages = pages;
1308
1309 return 0;
1310}
1311
1312I915_SELFTEST_EXPORTstatic int i915_vma_get_pages(struct i915_vma *vma)
1313{
1314 int err;
1315
1316 if (atomic_add_unless(&vma->pages_count, 1, 0))
1317 return 0;
1318
1319 err = i915_gem_object_pin_pages(vma->obj);
1320 if (err)
1321 return err;
1322
1323 err = __i915_vma_get_pages(vma);
1324 if (err)
1325 goto err_unpin;
1326
1327 vma->page_sizes = vma->obj->mm.page_sizes;
1328 atomic_inc(&vma->pages_count)__sync_fetch_and_add(&vma->pages_count, 1);
1329
1330 return 0;
1331
1332err_unpin:
1333 __i915_gem_object_unpin_pages(vma->obj);
1334
1335 return err;
1336}
1337
1338void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
1339{
1340 /*
1341 * Before we release the pages that were bound by this vma, we
1342 * must invalidate all the TLBs that may still have a reference
1343 * back to our physical address. It only needs to be done once,
1344 * so after updating the PTE to point away from the pages, record
1345 * the most recent TLB invalidation seqno, and if we have not yet
1346 * flushed the TLBs upon release, perform a full invalidation.
1347 */
1348 WRITE_ONCE(*tlb, intel_gt_next_invalidate_tlb_full(vm->gt))({ typeof(*tlb) __tmp = (intel_gt_next_invalidate_tlb_full(vm
->gt)); *(volatile typeof(*tlb) *)&(*tlb) = __tmp; __tmp
; })
;
1349}
1350
1351static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
1352{
1353 /* We allocate under vma_get_pages, so beware the shrinker */
1354 GEM_BUG_ON(atomic_read(&vma->pages_count) < count)((void)0);
1355
1356 if (atomic_sub_return(count, &vma->pages_count)__sync_sub_and_fetch(&vma->pages_count, count) == 0) {
1357 if (vma->pages != vma->obj->mm.pages) {
1358 sg_free_table(vma->pages);
1359 kfree(vma->pages);
1360 }
1361 vma->pages = NULL((void *)0);
1362
1363 i915_gem_object_unpin_pages(vma->obj);
1364 }
1365}
1366
1367I915_SELFTEST_EXPORTstatic void i915_vma_put_pages(struct i915_vma *vma)
1368{
1369 if (atomic_add_unless(&vma->pages_count, -1, 1))
1370 return;
1371
1372 __vma_put_pages(vma, 1);
1373}
1374
1375static void vma_unbind_pages(struct i915_vma *vma)
1376{
1377 unsigned int count;
1378
1379 lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0);
1380
1381 /* The upper portion of pages_count is the number of bindings */
1382 count = atomic_read(&vma->pages_count)({ typeof(*(&vma->pages_count)) __tmp = *(volatile typeof
(*(&vma->pages_count)) *)&(*(&vma->pages_count
)); membar_datadep_consumer(); __tmp; })
;
1383 count >>= I915_VMA_PAGES_BIAS24;
1384 GEM_BUG_ON(!count)((void)0);
1385
1386 __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS24);
1387}
1388
1389int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1390 u64 size, u64 alignment, u64 flags)
1391{
1392 struct i915_vma_work *work = NULL((void *)0);
1393 struct dma_fence *moving = NULL((void *)0);
1394 struct i915_vma_resource *vma_res = NULL((void *)0);
1395 intel_wakeref_t wakeref = 0;
1396 unsigned int bound;
1397 int err;
1398
1399 assert_vma_held(vma)do { (void)(&((vma)->obj->base.resv)->lock.base)
; } while(0)
;
1400 GEM_BUG_ON(!ww)((void)0);
1401
1402 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND)extern char _ctassert[(!((1ULL << (10)) != ((int)(1UL <<
(10))))) ? 1 : -1 ] __attribute__((__unused__))
;
1403 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND)extern char _ctassert[(!((1ULL << (11)) != ((int)(1UL <<
(11))))) ? 1 : -1 ] __attribute__((__unused__))
;
1404
1405 GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)))((void)0);
1406
1407 /* First try and grab the pin without rebinding the vma */
1408 if (try_qad_pin(vma, flags))
1409 return 0;
1410
1411 err = i915_vma_get_pages(vma);
1412 if (err)
1413 return err;
1414
1415 if (flags & PIN_GLOBAL(1ULL << (10)))
1416 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
1417
1418 if (flags & vma->vm->bind_async_flags) {
1419 /* lock VM */
1420 err = i915_vm_lock_objects(vma->vm, ww);
1421 if (err)
1422 goto err_rpm;
1423
1424 work = i915_vma_work();
1425 if (!work) {
1426 err = -ENOMEM12;
1427 goto err_rpm;
1428 }
1429
1430 work->vm = vma->vm;
1431
1432 err = i915_gem_object_get_moving_fence(vma->obj, &moving);
1433 if (err)
1434 goto err_rpm;
1435
1436 dma_fence_work_chain(&work->base, moving);
1437
1438 /* Allocate enough page directories to used PTE */
1439 if (vma->vm->allocate_va_range) {
1440 err = i915_vm_alloc_pt_stash(vma->vm,
1441 &work->stash,
1442 vma->size);
1443 if (err)
1444 goto err_fence;
1445
1446 err = i915_vm_map_pt_stash(vma->vm, &work->stash);
1447 if (err)
1448 goto err_fence;
1449 }
1450 }
1451
1452 vma_res = i915_vma_resource_alloc();
1453 if (IS_ERR(vma_res)) {
1454 err = PTR_ERR(vma_res);
1455 goto err_fence;
1456 }
1457
1458 /*
1459 * Differentiate between user/kernel vma inside the aliasing-ppgtt.
1460 *
1461 * We conflate the Global GTT with the user's vma when using the
1462 * aliasing-ppgtt, but it is still vitally important to try and
1463 * keep the use cases distinct. For example, userptr objects are
1464 * not allowed inside the Global GTT as that will cause lock
1465 * inversions when we have to evict them the mmu_notifier callbacks -
1466 * but they are allowed to be part of the user ppGTT which can never
1467 * be mapped. As such we try to give the distinct users of the same
1468 * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
1469 * and i915_ppgtt separate].
1470 *
1471 * NB this may cause us to mask real lock inversions -- while the
1472 * code is safe today, lockdep may not be able to spot future
1473 * transgressions.
1474 */
1475 err = mutex_lock_interruptible_nested(&vma->vm->mutex,mutex_lock_interruptible(&vma->vm->mutex)
1476 !(flags & PIN_GLOBAL))mutex_lock_interruptible(&vma->vm->mutex);
1477 if (err)
1478 goto err_vma_res;
1479
1480 /* No more allocations allowed now we hold vm->mutex */
1481
1482 if (unlikely(i915_vma_is_closed(vma))__builtin_expect(!!(i915_vma_is_closed(vma)), 0)) {
1483 err = -ENOENT2;
1484 goto err_unlock;
1485 }
1486
1487 bound = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*(
&vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer
(); __tmp; })
;
1488 if (unlikely(bound & I915_VMA_ERROR)__builtin_expect(!!(bound & ((int)(1UL << (12)))), 0
)
) {
1489 err = -ENOMEM12;
1490 goto err_unlock;
1491 }
1492
1493 if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))__builtin_expect(!!(!((bound + 1) & 0x3ff)), 0)) {
1494 err = -EAGAIN35; /* pins are meant to be fairly temporary */
1495 goto err_unlock;
1496 }
1497
1498 if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))__builtin_expect(!!(!(flags & ~bound & (((int)(1UL <<
(10))) | ((int)(1UL << (11)))))), 0)
) {
1499 if (!(flags & PIN_VALIDATE(1ULL << (8))))
1500 __i915_vma_pin(vma);
1501 goto err_unlock;
1502 }
1503
1504 err = i915_active_acquire(&vma->active);
1505 if (err)
1506 goto err_unlock;
1507
1508 if (!(bound & I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) {
1509 err = i915_vma_insert(vma, ww, size, alignment, flags);
1510 if (err)
1511 goto err_active;
1512
1513 if (i915_is_ggtt(vma->vm)((vma->vm)->is_ggtt))
1514 __i915_vma_set_map_and_fenceable(vma);
1515 }
1516
1517 GEM_BUG_ON(!vma->pages)((void)0);
1518 err = i915_vma_bind(vma,
1519 vma->obj->cache_level,
1520 flags, work, vma_res);
1521 vma_res = NULL((void *)0);
1522 if (err)
1523 goto err_remove;
1524
1525 /* There should only be at most 2 active bindings (user, global) */
1526 GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound)((void)0);
1527 atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count)__sync_fetch_and_add(&vma->pages_count, ((1UL <<
(24)) | 1))
;
1528 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1529
1530 if (!(flags & PIN_VALIDATE(1ULL << (8)))) {
1531 __i915_vma_pin(vma);
1532 GEM_BUG_ON(!i915_vma_is_pinned(vma))((void)0);
1533 }
1534 GEM_BUG_ON(!i915_vma_is_bound(vma, flags))((void)0);
1535 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags))((void)0);
1536
1537err_remove:
1538 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) {
1539 i915_vma_detach(vma);
1540 drm_mm_remove_node(&vma->node);
1541 }
1542err_active:
1543 i915_active_release(&vma->active);
1544err_unlock:
1545 mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex);
1546err_vma_res:
1547 i915_vma_resource_free(vma_res);
1548err_fence:
1549 if (work)
1550 dma_fence_work_commit_imm(&work->base);
1551err_rpm:
1552 if (wakeref)
1553 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
1554
1555 if (moving)
1556 dma_fence_put(moving);
1557
1558 i915_vma_put_pages(vma);
1559 return err;
1560}
1561
1562static void flush_idle_contexts(struct intel_gt *gt)
1563{
1564 struct intel_engine_cs *engine;
1565 enum intel_engine_id id;
1566
1567 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else
1568 intel_engine_flush_barriers(engine);
1569
1570 intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT(0x7fffffff));
1571}
1572
1573static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1574 u32 align, unsigned int flags)
1575{
1576 struct i915_address_space *vm = vma->vm;
1577 int err;
1578
1579 do {
1580 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL(1ULL << (10)));
1581
1582 if (err != -ENOSPC28) {
1583 if (!err) {
1584 err = i915_vma_wait_for_bind(vma);
1585 if (err)
1586 i915_vma_unpin(vma);
1587 }
1588 return err;
1589 }
1590
1591 /* Unlike i915_vma_pin, we don't take no for an answer! */
1592 flush_idle_contexts(vm->gt);
1593 if (mutex_lock_interruptible(&vm->mutex) == 0) {
1594 /*
1595 * We pass NULL ww here, as we don't want to unbind
1596 * locked objects when called from execbuf when pinning
1597 * is removed. This would probably regress badly.
1598 */
1599 i915_gem_evict_vm(vm, NULL((void *)0), NULL((void *)0));
1600 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
1601 }
1602 } while (1);
1603}
1604
1605int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1606 u32 align, unsigned int flags)
1607{
1608 struct i915_gem_ww_ctx _ww;
1609 int err;
1610
1611 GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0);
1612
1613 if (ww)
1614 return __i915_ggtt_pin(vma, ww, align, flags);
1615
1616 lockdep_assert_not_held(&vma->obj->base.resv->lock.base)do { (void)(&vma->obj->base.resv->lock.base); } while
(0)
;
1617
1618 for_i915_gem_ww(&_ww, err, true)for (i915_gem_ww_ctx_init(&_ww, 1), (err) = -11; (err) ==
-11; (err) = __i915_gem_ww_fini(&_ww, err))
{
1619 err = i915_gem_object_lock(vma->obj, &_ww);
1620 if (!err)
1621 err = __i915_ggtt_pin(vma, &_ww, align, flags);
1622 }
1623
1624 return err;
1625}
1626
1627static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1628{
1629 /*
1630 * We defer actually closing, unbinding and destroying the VMA until
1631 * the next idle point, or if the object is freed in the meantime. By
1632 * postponing the unbind, we allow for it to be resurrected by the
1633 * client, avoiding the work required to rebind the VMA. This is
1634 * advantageous for DRI, where the client/server pass objects
1635 * between themselves, temporarily opening a local VMA to the
1636 * object, and then closing it again. The same object is then reused
1637 * on the next frame (or two, depending on the depth of the swap queue)
1638 * causing us to rebind the VMA once more. This ends up being a lot
1639 * of wasted work for the steady state.
1640 */
1641 GEM_BUG_ON(i915_vma_is_closed(vma))((void)0);
1642 list_add(&vma->closed_link, &gt->closed_vma);
1643}
1644
1645void i915_vma_close(struct i915_vma *vma)
1646{
1647 struct intel_gt *gt = vma->vm->gt;
1648 unsigned long flags;
1649
1650 if (i915_vma_is_ggtt(vma))
1651 return;
1652
1653 GEM_BUG_ON(!atomic_read(&vma->open_count))((void)0);
1654 if (atomic_dec_and_lock_irqsave(&vma->open_count,atomic_dec_and_lock(&vma->open_count, &gt->closed_lock
)
1655 &gt->closed_lock,atomic_dec_and_lock(&vma->open_count, &gt->closed_lock
)
1656 flags)atomic_dec_and_lock(&vma->open_count, &gt->closed_lock
)
) {
1657 __vma_close(vma, gt);
1658 spin_unlock_irqrestore(&gt->closed_lock, flags)do { (void)(flags); mtx_leave(&gt->closed_lock); } while
(0)
;
1659 }
1660}
1661
1662static void __i915_vma_remove_closed(struct i915_vma *vma)
1663{
1664 list_del_init(&vma->closed_link);
1665}
1666
1667void i915_vma_reopen(struct i915_vma *vma)
1668{
1669 struct intel_gt *gt = vma->vm->gt;
1670
1671 spin_lock_irq(&gt->closed_lock)mtx_enter(&gt->closed_lock);
1672 if (i915_vma_is_closed(vma))
1673 __i915_vma_remove_closed(vma);
1674 spin_unlock_irq(&gt->closed_lock)mtx_leave(&gt->closed_lock);
1675}
1676
1677static void force_unbind(struct i915_vma *vma)
1678{
1679 if (!drm_mm_node_allocated(&vma->node))
1680 return;
1681
1682 atomic_and(~I915_VMA_PIN_MASK, &vma->flags)__sync_fetch_and_and(&vma->flags, ~0x3ff);
1683 WARN_ON(__i915_vma_unbind(vma))({ int __ret = !!(__i915_vma_unbind(vma)); if (__ret) printf(
"WARNING %s failed at %s:%d\n", "__i915_vma_unbind(vma)", "/usr/src/sys/dev/pci/drm/i915/i915_vma.c"
, 1683); __builtin_expect(!!(__ret), 0); })
;
1684 GEM_BUG_ON(drm_mm_node_allocated(&vma->node))((void)0);
1685}
1686
1687static void release_references(struct i915_vma *vma, struct intel_gt *gt,
1688 bool_Bool vm_ddestroy)
1689{
1690 struct drm_i915_gem_object *obj = vma->obj;
1691
1692 GEM_BUG_ON(i915_vma_is_active(vma))((void)0);
1693
1694 spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
1695 list_del(&vma->obj_link);
1696 if (!RB_EMPTY_NODE(&vma->obj_node)((&vma->obj_node)->__entry.rbe_parent == &vma->
obj_node)
)
1697 rb_erase(&vma->obj_node, &obj->vma.tree)linux_root_RB_REMOVE((struct linux_root *)(&obj->vma.tree
), (&vma->obj_node))
;
1698
1699 spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
1700
1701 spin_lock_irq(&gt->closed_lock)mtx_enter(&gt->closed_lock);
1702 __i915_vma_remove_closed(vma);
1703 spin_unlock_irq(&gt->closed_lock)mtx_leave(&gt->closed_lock);
1704
1705 if (vm_ddestroy)
1706 i915_vm_resv_put(vma->vm);
1707
1708 i915_active_fini(&vma->active);
1709 GEM_WARN_ON(vma->resource)({ __builtin_expect(!!(!!(vma->resource)), 0); });
1710 i915_vma_free(vma);
1711}
1712
1713/**
1714 * i915_vma_destroy_locked - Remove all weak reference to the vma and put
1715 * the initial reference.
1716 *
1717 * This function should be called when it's decided the vma isn't needed
1718 * anymore. The caller must assure that it doesn't race with another lookup
1719 * plus destroy, typically by taking an appropriate reference.
1720 *
1721 * Current callsites are
1722 * - __i915_gem_object_pages_fini()
1723 * - __i915_vm_close() - Blocks the above function by taking a reference on
1724 * the object.
1725 * - __i915_vma_parked() - Blocks the above functions by taking a reference
1726 * on the vm and a reference on the object. Also takes the object lock so
1727 * destruction from __i915_vma_parked() can be blocked by holding the
1728 * object lock. Since the object lock is only allowed from within i915 with
1729 * an object refcount, holding the object lock also implicitly blocks the
1730 * vma freeing from __i915_gem_object_pages_fini().
1731 *
1732 * Because of locks taken during destruction, a vma is also guaranteed to
1733 * stay alive while the following locks are held if it was looked up while
1734 * holding one of the locks:
1735 * - vm->mutex
1736 * - obj->vma.lock
1737 * - gt->closed_lock
1738 */
1739void i915_vma_destroy_locked(struct i915_vma *vma)
1740{
1741 lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0);
1742
1743 force_unbind(vma);
1744 list_del_init(&vma->vm_link);
1745 release_references(vma, vma->vm->gt, false0);
1746}
1747
1748void i915_vma_destroy(struct i915_vma *vma)
1749{
1750 struct intel_gt *gt;
1751 bool_Bool vm_ddestroy;
1752
1753 mutex_lock(&vma->vm->mutex)rw_enter_write(&vma->vm->mutex);
1754 force_unbind(vma);
1755 list_del_init(&vma->vm_link);
1756 vm_ddestroy = vma->vm_ddestroy;
1757 vma->vm_ddestroy = false0;
1758
1759 /* vma->vm may be freed when releasing vma->vm->mutex. */
1760 gt = vma->vm->gt;
1761 mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex);
1762 release_references(vma, gt, vm_ddestroy);
1763}
1764
1765void i915_vma_parked(struct intel_gt *gt)
1766{
1767 struct i915_vma *vma, *next;
1768 DRM_LIST_HEAD(closed)struct list_head closed = { &(closed), &(closed) };
1769
1770 spin_lock_irq(&gt->closed_lock)mtx_enter(&gt->closed_lock);
1771 list_for_each_entry_safe(vma, next, &gt->closed_vma, closed_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = ((&gt->closed_vma)->next); (__typeof(*
vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link
) );}), next = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = (vma->closed_link.next); (__typeof(*vma) *)( (
char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link
) );}); &vma->closed_link != (&gt->closed_vma);
vma = next, next = ({ const __typeof( ((__typeof(*next) *)0)
->closed_link ) *__mptr = (next->closed_link.next); (__typeof
(*next) *)( (char *)__mptr - __builtin_offsetof(__typeof(*next
), closed_link) );}))
{
1772 struct drm_i915_gem_object *obj = vma->obj;
1773 struct i915_address_space *vm = vma->vm;
1774
1775 /* XXX All to avoid keeping a reference on i915_vma itself */
1776
1777 if (!kref_get_unless_zero(&obj->base.refcount))
1778 continue;
1779
1780 if (!i915_vm_tryget(vm)) {
1781 i915_gem_object_put(obj);
1782 continue;
1783 }
1784
1785 list_move(&vma->closed_link, &closed);
1786 }
1787 spin_unlock_irq(&gt->closed_lock)mtx_leave(&gt->closed_lock);
1788
1789 /* As the GT is held idle, no vma can be reopened as we destroy them */
1790 list_for_each_entry_safe(vma, next, &closed, closed_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = ((&closed)->next); (__typeof(*vma) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*vma), closed_link) )
;}), next = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link
) *__mptr = (vma->closed_link.next); (__typeof(*vma) *)( (
char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link
) );}); &vma->closed_link != (&closed); vma = next
, next = ({ const __typeof( ((__typeof(*next) *)0)->closed_link
) *__mptr = (next->closed_link.next); (__typeof(*next) *)
( (char *)__mptr - __builtin_offsetof(__typeof(*next), closed_link
) );}))
{
1791 struct drm_i915_gem_object *obj = vma->obj;
1792 struct i915_address_space *vm = vma->vm;
1793
1794 if (i915_gem_object_trylock(obj, NULL((void *)0))) {
1795 INIT_LIST_HEAD(&vma->closed_link);
1796 i915_vma_destroy(vma);
1797 i915_gem_object_unlock(obj);
1798 } else {
1799 /* back you go.. */
1800 spin_lock_irq(&gt->closed_lock)mtx_enter(&gt->closed_lock);
1801 list_add(&vma->closed_link, &gt->closed_vma);
1802 spin_unlock_irq(&gt->closed_lock)mtx_leave(&gt->closed_lock);
1803 }
1804
1805 i915_gem_object_put(obj);
1806 i915_vm_put(vm);
1807 }
1808}
1809
1810static void __i915_vma_iounmap(struct i915_vma *vma)
1811{
1812 GEM_BUG_ON(i915_vma_is_pinned(vma))((void)0);
1813
1814 if (vma->iomap == NULL((void *)0))
1815 return;
1816
1817 if (page_unmask_bits(vma->iomap)((unsigned long)(vma->iomap) & ((1UL << (12)) - 1
))
)
1818 __i915_gem_object_release_map(vma->obj);
1819 else {
1820#ifdef __linux__
1821 io_mapping_unmap(vma->iomap);
1822#else
1823 struct drm_i915_privateinteldrm_softc *dev_priv = vma->vm->i915;
1824 agp_unmap_subregion(dev_priv->agph, vma->bsh, vma->node.size);
1825#endif
1826 }
1827 vma->iomap = NULL((void *)0);
1828}
1829
1830void i915_vma_revoke_mmap(struct i915_vma *vma)
1831{
1832 struct drm_vma_offset_node *node;
1833 u64 vma_offset;
1834
1835 if (!i915_vma_has_userfault(vma))
1836 return;
1837
1838 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma))((void)0);
1839 GEM_BUG_ON(!vma->obj->userfault_count)((void)0);
1840
1841 node = &vma->mmo->vma_node;
Value stored to 'node' is never read
1842 vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT12;
1843#ifdef __linux__
1844 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
1845 drm_vma_node_offset_addr(node) + vma_offset,
1846 vma->size,
1847 1);
1848#else
1849 struct drm_i915_privateinteldrm_softc *dev_priv = vma->obj->base.dev->dev_private;
1850 struct vm_page *pg;
1851
1852 for (pg = &dev_priv->pgs[atop(vma->node.start)((vma->node.start) >> 12)];
1853 pg != &dev_priv->pgs[atop(vma->node.start + vma->size)((vma->node.start + vma->size) >> 12)];
1854 pg++)
1855 pmap_page_protect(pg, PROT_NONE0x00);
1856#endif
1857
1858 i915_vma_unset_userfault(vma);
1859 if (!--vma->obj->userfault_count)
1860 list_del(&vma->obj->userfault_link);
1861}
1862
1863static int
1864__i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1865{
1866 return __i915_request_await_exclusive(rq, &vma->active);
1867}
1868
1869static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1870{
1871 int err;
1872
1873 /* Wait for the vma to be bound before we start! */
1874 err = __i915_request_await_bind(rq, vma);
1875 if (err)
1876 return err;
1877
1878 return i915_active_add_request(&vma->active, rq);
1879}
1880
1881int _i915_vma_move_to_active(struct i915_vma *vma,
1882 struct i915_request *rq,
1883 struct dma_fence *fence,
1884 unsigned int flags)
1885{
1886 struct drm_i915_gem_object *obj = vma->obj;
1887 int err;
1888
1889 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0)
;
1890
1891 GEM_BUG_ON(!vma->pages)((void)0);
1892
1893 err = __i915_vma_move_to_active(vma, rq);
1894 if (unlikely(err)__builtin_expect(!!(err), 0))
1895 return err;
1896
1897 /*
1898 * Reserve fences slot early to prevent an allocation after preparing
1899 * the workload and associating fences with dma_resv.
1900 */
1901 if (fence && !(flags & __EXEC_OBJECT_NO_RESERVE(1UL << (31)))) {
1902 struct dma_fence *curr;
1903 int idx;
1904
1905 dma_fence_array_for_each(curr, idx, fence)for (idx = 0, curr = dma_fence_array_first(fence); curr != ((
void *)0); ++(idx), curr = dma_fence_array_next(fence, idx))
1906 ;
1907 err = dma_resv_reserve_fences(vma->obj->base.resv, idx);
1908 if (unlikely(err)__builtin_expect(!!(err), 0))
1909 return err;
1910 }
1911
1912 if (flags & EXEC_OBJECT_WRITE(1<<2)) {
1913 struct intel_frontbuffer *front;
1914
1915 front = __intel_frontbuffer_get(obj);
1916 if (unlikely(front)__builtin_expect(!!(front), 0)) {
1917 if (intel_frontbuffer_invalidate(front, ORIGIN_CS))
1918 i915_active_add_request(&front->write, rq);
1919 intel_frontbuffer_put(front);
1920 }
1921 }
1922
1923 if (fence) {
1924 struct dma_fence *curr;
1925 enum dma_resv_usage usage;
1926 int idx;
1927
1928 if (flags & EXEC_OBJECT_WRITE(1<<2)) {
1929 usage = DMA_RESV_USAGE_WRITE;
1930 obj->write_domain = I915_GEM_DOMAIN_RENDER0x00000002;
1931 obj->read_domains = 0;
1932 } else {
1933 usage = DMA_RESV_USAGE_READ;
1934 obj->write_domain = 0;
1935 }
1936
1937 dma_fence_array_for_each(curr, idx, fence)for (idx = 0, curr = dma_fence_array_first(fence); curr != ((
void *)0); ++(idx), curr = dma_fence_array_next(fence, idx))
1938 dma_resv_add_fence(vma->obj->base.resv, curr, usage);
1939 }
1940
1941 if (flags & EXEC_OBJECT_NEEDS_FENCE(1<<0) && vma->fence)
1942 i915_active_add_request(&vma->fence->active, rq);
1943
1944 obj->read_domains |= I915_GEM_GPU_DOMAINS(0x00000002 | 0x00000004 | 0x00000008 | 0x00000010 | 0x00000020
)
;
1945 obj->mm.dirty = true1;
1946
1947 GEM_BUG_ON(!i915_vma_is_active(vma))((void)0);
1948 return 0;
1949}
1950
1951struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool_Bool async)
1952{
1953 struct i915_vma_resource *vma_res = vma->resource;
1954 struct dma_fence *unbind_fence;
1955
1956 GEM_BUG_ON(i915_vma_is_pinned(vma))((void)0);
1957 assert_vma_held_evict(vma);
1958
1959 if (i915_vma_is_map_and_fenceable(vma)) {
1960 /* Force a pagefault for domain tracking on next user access */
1961 i915_vma_revoke_mmap(vma);
1962
1963 /*
1964 * Check that we have flushed all writes through the GGTT
1965 * before the unbind, other due to non-strict nature of those
1966 * indirect writes they may end up referencing the GGTT PTE
1967 * after the unbind.
1968 *
1969 * Note that we may be concurrently poking at the GGTT_WRITE
1970 * bit from set-domain, as we mark all GGTT vma associated
1971 * with an object. We know this is for another vma, as we
1972 * are currently unbinding this one -- so if this vma will be
1973 * reused, it will be refaulted and have its dirty bit set
1974 * before the next write.
1975 */
1976 i915_vma_flush_writes(vma);
1977
1978 /* release the fence reg _after_ flushing */
1979 i915_vma_revoke_fence(vma);
1980
1981 clear_bit(I915_VMA_CAN_FENCE_BIT14, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags));
1982 }
1983
1984 __i915_vma_iounmap(vma);
1985
1986 GEM_BUG_ON(vma->fence)((void)0);
1987 GEM_BUG_ON(i915_vma_has_userfault(vma))((void)0);
1988
1989 /* Object backend must be async capable. */
1990 GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt)({ __builtin_expect(!!(!!(async && !vma->resource->
bi.pages_rsgt)), 0); })
;
1991
1992 /* If vm is not open, unbind is a nop. */
1993 vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10)))) &&
1994 kref_read(&vma->vm->ref);
1995 vma_res->skip_pte_rewrite = !kref_read(&vma->vm->ref) ||
1996 vma->vm->skip_pte_rewrite;
1997 trace_i915_vma_unbind(vma);
1998
1999 if (async)
2000 unbind_fence = i915_vma_resource_unbind(vma_res,
2001 &vma->obj->mm.tlb);
2002 else
2003 unbind_fence = i915_vma_resource_unbind(vma_res, NULL((void *)0));
2004
2005 vma->resource = NULL((void *)0);
2006
2007 atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),__sync_fetch_and_and(&vma->flags, ~((((int)(1UL <<
(10))) | ((int)(1UL << (11)))) | ((int)(1UL << (
12))) | ((int)(1UL << (16)))))
2008 &vma->flags)__sync_fetch_and_and(&vma->flags, ~((((int)(1UL <<
(10))) | ((int)(1UL << (11)))) | ((int)(1UL << (
12))) | ((int)(1UL << (16)))))
;
2009
2010 i915_vma_detach(vma);
2011
2012 if (!async) {
2013 if (unbind_fence) {
2014 dma_fence_wait(unbind_fence, false0);
2015 dma_fence_put(unbind_fence);
2016 unbind_fence = NULL((void *)0);
2017 }
2018 vma_invalidate_tlb(vma->vm, &vma->obj->mm.tlb);
2019 }
2020
2021 /*
2022 * Binding itself may not have completed until the unbind fence signals,
2023 * so don't drop the pages until that happens, unless the resource is
2024 * async_capable.
2025 */
2026
2027 vma_unbind_pages(vma);
2028 return unbind_fence;
2029}
2030
2031int __i915_vma_unbind(struct i915_vma *vma)
2032{
2033 int ret;
2034
2035 lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0);
2036 assert_vma_held_evict(vma);
2037
2038 if (!drm_mm_node_allocated(&vma->node))
2039 return 0;
2040
2041 if (i915_vma_is_pinned(vma)) {
2042 vma_print_allocator(vma, "is pinned");
2043 return -EAGAIN35;
2044 }
2045
2046 /*
2047 * After confirming that no one else is pinning this vma, wait for
2048 * any laggards who may have crept in during the wait (through
2049 * a residual pin skipping the vm->mutex) to complete.
2050 */
2051 ret = i915_vma_sync(vma);
2052 if (ret)
2053 return ret;
2054
2055 GEM_BUG_ON(i915_vma_is_active(vma))((void)0);
2056 __i915_vma_evict(vma, false0);
2057
2058 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
2059 return 0;
2060}
2061
2062static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma)
2063{
2064 struct dma_fence *fence;
2065
2066 lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0);
2067
2068 if (!drm_mm_node_allocated(&vma->node))
2069 return NULL((void *)0);
2070
2071 if (i915_vma_is_pinned(vma) ||
2072 &vma->obj->mm.rsgt->table != vma->resource->bi.pages)
2073 return ERR_PTR(-EAGAIN35);
2074
2075 /*
2076 * We probably need to replace this with awaiting the fences of the
2077 * object's dma_resv when the vma active goes away. When doing that
2078 * we need to be careful to not add the vma_resource unbind fence
2079 * immediately to the object's dma_resv, because then unbinding
2080 * the next vma from the object, in case there are many, will
2081 * actually await the unbinding of the previous vmas, which is
2082 * undesirable.
2083 */
2084 if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active,
2085 I915_ACTIVE_AWAIT_EXCL(1UL << (0)) |
2086 I915_ACTIVE_AWAIT_ACTIVE(1UL << (1))) < 0) {
2087 return ERR_PTR(-EBUSY16);
2088 }
2089
2090 fence = __i915_vma_evict(vma, true1);
2091
2092 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
2093
2094 return fence;
2095}
2096
2097int i915_vma_unbind(struct i915_vma *vma)
2098{
2099 struct i915_address_space *vm = vma->vm;
2100 intel_wakeref_t wakeref = 0;
2101 int err;
2102
2103 assert_object_held_shared(vma->obj);
2104
2105 /* Optimistic wait before taking the mutex */
2106 err = i915_vma_sync(vma);
2107 if (err)
2108 return err;
2109
2110 if (!drm_mm_node_allocated(&vma->node))
2111 return 0;
2112
2113 if (i915_vma_is_pinned(vma)) {
2114 vma_print_allocator(vma, "is pinned");
2115 return -EAGAIN35;
2116 }
2117
2118 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10)))))
2119 /* XXX not always required: nop_clear_range */
2120 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
2121
2122 err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref)mutex_lock_interruptible(&vma->vm->mutex);
2123 if (err)
2124 goto out_rpm;
2125
2126 err = __i915_vma_unbind(vma);
2127 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
2128
2129out_rpm:
2130 if (wakeref)
2131 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
2132 return err;
2133}
2134
2135int i915_vma_unbind_async(struct i915_vma *vma, bool_Bool trylock_vm)
2136{
2137 struct drm_i915_gem_object *obj = vma->obj;
2138 struct i915_address_space *vm = vma->vm;
2139 intel_wakeref_t wakeref = 0;
2140 struct dma_fence *fence;
2141 int err;
2142
2143 /*
2144 * We need the dma-resv lock since we add the
2145 * unbind fence to the dma-resv object.
2146 */
2147 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0)
;
2148
2149 if (!drm_mm_node_allocated(&vma->node))
2150 return 0;
2151
2152 if (i915_vma_is_pinned(vma)) {
2153 vma_print_allocator(vma, "is pinned");
2154 return -EAGAIN35;
2155 }
2156
2157 if (!obj->mm.rsgt)
2158 return -EBUSY16;
2159
2160 err = dma_resv_reserve_fences(obj->base.resv, 2);
2161 if (err)
2162 return -EBUSY16;
2163
2164 /*
2165 * It would be great if we could grab this wakeref from the
2166 * async unbind work if needed, but we can't because it uses
2167 * kmalloc and it's in the dma-fence signalling critical path.
2168 */
2169 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10)))))
2170 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
2171
2172 if (trylock_vm && !mutex_trylock(&vm->mutex)(rw_enter(&vm->mutex, 0x0001UL | 0x0040UL) == 0)) {
2173 err = -EBUSY16;
2174 goto out_rpm;
2175 } else if (!trylock_vm) {
2176 err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref)mutex_lock_interruptible(&vm->mutex);
2177 if (err)
2178 goto out_rpm;
2179 }
2180
2181 fence = __i915_vma_unbind_async(vma);
2182 mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex);
2183 if (IS_ERR_OR_NULL(fence)) {
2184 err = PTR_ERR_OR_ZERO(fence);
2185 goto out_rpm;
2186 }
2187
2188 dma_resv_add_fence(obj->base.resv, fence, DMA_RESV_USAGE_READ);
2189 dma_fence_put(fence);
2190
2191out_rpm:
2192 if (wakeref)
2193 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
2194 return err;
2195}
2196
2197int i915_vma_unbind_unlocked(struct i915_vma *vma)
2198{
2199 int err;
2200
2201 i915_gem_object_lock(vma->obj, NULL((void *)0));
2202 err = i915_vma_unbind(vma);
2203 i915_gem_object_unlock(vma->obj);
2204
2205 return err;
2206}
2207
2208struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
2209{
2210 i915_gem_object_make_unshrinkable(vma->obj);
2211 return vma;
2212}
2213
2214void i915_vma_make_shrinkable(struct i915_vma *vma)
2215{
2216 i915_gem_object_make_shrinkable(vma->obj);
2217}
2218
2219void i915_vma_make_purgeable(struct i915_vma *vma)
2220{
2221 i915_gem_object_make_purgeable(vma->obj);
2222}
2223
2224#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
2225#include "selftests/i915_vma.c"
2226#endif
2227
2228void i915_vma_module_exit(void)
2229{
2230#ifdef __linux__
2231 kmem_cache_destroy(slab_vmas);
2232#else
2233 pool_destroy(&slab_vmas);
2234#endif
2235}
2236
2237int __init i915_vma_module_init(void)
2238{
2239#ifdef __linux__
2240 slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
2241 if (!slab_vmas)
2242 return -ENOMEM12;
2243#else
2244 pool_init(&slab_vmas, sizeof(struct i915_vma),
2245 CACHELINESIZE64, IPL_NONE0x0, 0, "drmvma", NULL((void *)0));
2246#endif
2247
2248 return 0;
2249}