File: | dev/pci/drm/i915/i915_vma.c |
Warning: | line 1217, column 2 Value stored to 'vma_offset' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* |
2 | * Copyright © 2016 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #include <linux/sched/mm.h> |
26 | #include <drm/drm_gem.h> |
27 | |
28 | #include "display/intel_frontbuffer.h" |
29 | |
30 | #include "gt/intel_engine.h" |
31 | #include "gt/intel_engine_heartbeat.h" |
32 | #include "gt/intel_gt.h" |
33 | #include "gt/intel_gt_requests.h" |
34 | |
35 | #include "i915_drv.h" |
36 | #include "i915_globals.h" |
37 | #include "i915_sw_fence_work.h" |
38 | #include "i915_trace.h" |
39 | #include "i915_vma.h" |
40 | |
41 | #include <dev/pci/agpvar.h> |
42 | |
43 | static struct i915_global_vma { |
44 | struct i915_global base; |
45 | #ifdef __linux__ |
46 | struct kmem_cache *slab_vmas; |
47 | #else |
48 | struct pool slab_vmas; |
49 | #endif |
50 | } global; |
51 | |
52 | struct i915_vma *i915_vma_alloc(void) |
53 | { |
54 | #ifdef __linux__ |
55 | return kmem_cache_zalloc(global.slab_vmas, GFP_KERNEL(0x0001 | 0x0004)); |
56 | #else |
57 | return pool_get(&global.slab_vmas, PR_WAITOK0x0001 | PR_ZERO0x0008); |
58 | #endif |
59 | } |
60 | |
61 | void i915_vma_free(struct i915_vma *vma) |
62 | { |
63 | #ifdef __linux__ |
64 | return kmem_cache_free(global.slab_vmas, vma); |
65 | #else |
66 | pool_put(&global.slab_vmas, vma); |
67 | #endif |
68 | } |
69 | |
70 | #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM)0 && IS_ENABLED(CONFIG_DRM_DEBUG_MM)0 |
71 | |
72 | #include <linux/stackdepot.h> |
73 | |
74 | static void vma_print_allocator(struct i915_vma *vma, const char *reason) |
75 | { |
76 | unsigned long *entries; |
77 | unsigned int nr_entries; |
78 | char buf[512]; |
79 | |
80 | if (!vma->node.stack) { |
81 | DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: unknown owner\n" , vma->node.start, vma->node.size, reason) |
82 | vma->node.start, vma->node.size, reason)__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: unknown owner\n" , vma->node.start, vma->node.size, reason); |
83 | return; |
84 | } |
85 | |
86 | nr_entries = stack_depot_fetch(vma->node.stack, &entries); |
87 | stack_trace_snprint(buf, sizeof(buf), entries, nr_entries, 0); |
88 | DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: inserted at %s\n" , vma->node.start, vma->node.size, reason, buf) |
89 | vma->node.start, vma->node.size, reason, buf)__drm_dbg(DRM_UT_DRIVER, "vma.node [%08llx + %08llx] %s: inserted at %s\n" , vma->node.start, vma->node.size, reason, buf); |
90 | } |
91 | |
92 | #else |
93 | |
94 | static void vma_print_allocator(struct i915_vma *vma, const char *reason) |
95 | { |
96 | } |
97 | |
98 | #endif |
99 | |
100 | static inline struct i915_vma *active_to_vma(struct i915_active *ref) |
101 | { |
102 | return container_of(ref, typeof(struct i915_vma), active)({ const __typeof( ((typeof(struct i915_vma) *)0)->active ) *__mptr = (ref); (typeof(struct i915_vma) *)( (char *)__mptr - __builtin_offsetof(typeof(struct i915_vma), active) );}); |
103 | } |
104 | |
105 | static int __i915_vma_active(struct i915_active *ref) |
106 | { |
107 | return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT2; |
108 | } |
109 | |
110 | __i915_active_call__attribute__((__aligned__(4))) |
111 | static void __i915_vma_retire(struct i915_active *ref) |
112 | { |
113 | i915_vma_put(active_to_vma(ref)); |
114 | } |
115 | |
116 | static struct i915_vma * |
117 | vma_create(struct drm_i915_gem_object *obj, |
118 | struct i915_address_space *vm, |
119 | const struct i915_ggtt_view *view) |
120 | { |
121 | struct i915_vma *pos = ERR_PTR(-E2BIG7); |
122 | struct i915_vma *vma; |
123 | struct rb_node *rb, **p; |
124 | |
125 | /* The aliasing_ppgtt should never be used directly! */ |
126 | GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm)((void)0); |
127 | |
128 | vma = i915_vma_alloc(); |
129 | if (vma == NULL((void *)0)) |
130 | return ERR_PTR(-ENOMEM12); |
131 | |
132 | kref_init(&vma->ref); |
133 | rw_init(&vma->pages_mutex, "vmapg")_rw_init_flags(&vma->pages_mutex, "vmapg", 0, ((void * )0)); |
134 | vma->vm = i915_vm_get(vm); |
135 | vma->ops = &vm->vma_ops; |
136 | vma->obj = obj; |
137 | vma->resv = obj->base.resv; |
138 | vma->size = obj->base.size; |
139 | vma->display_alignment = I915_GTT_MIN_ALIGNMENT(1ULL << (12)); |
140 | |
141 | i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire)do { static struct lock_class_key __mkey; static struct lock_class_key __wkey; __i915_active_init(&vma->active, __i915_vma_active , __i915_vma_retire, &__mkey, &__wkey); } while (0); |
142 | |
143 | #ifdef notyet |
144 | /* Declare ourselves safe for use inside shrinkers */ |
145 | if (IS_ENABLED(CONFIG_LOCKDEP)0) { |
146 | fs_reclaim_acquire(GFP_KERNEL(0x0001 | 0x0004)); |
147 | might_lock(&vma->active.mutex); |
148 | fs_reclaim_release(GFP_KERNEL(0x0001 | 0x0004)); |
149 | } |
150 | #endif |
151 | |
152 | INIT_LIST_HEAD(&vma->closed_link); |
153 | |
154 | if (view && view->type != I915_GGTT_VIEW_NORMAL) { |
155 | vma->ggtt_view = *view; |
156 | if (view->type == I915_GGTT_VIEW_PARTIAL) { |
157 | GEM_BUG_ON(range_overflows_t(u64,((void)0) |
158 | view->partial.offset,((void)0) |
159 | view->partial.size,((void)0) |
160 | obj->base.size >> PAGE_SHIFT))((void)0); |
161 | vma->size = view->partial.size; |
162 | vma->size <<= PAGE_SHIFT12; |
163 | GEM_BUG_ON(vma->size > obj->base.size)((void)0); |
164 | } else if (view->type == I915_GGTT_VIEW_ROTATED) { |
165 | vma->size = intel_rotation_info_size(&view->rotated); |
166 | vma->size <<= PAGE_SHIFT12; |
167 | } else if (view->type == I915_GGTT_VIEW_REMAPPED) { |
168 | vma->size = intel_remapped_info_size(&view->remapped); |
169 | vma->size <<= PAGE_SHIFT12; |
170 | } |
171 | } |
172 | |
173 | if (unlikely(vma->size > vm->total)__builtin_expect(!!(vma->size > vm->total), 0)) |
174 | goto err_vma; |
175 | |
176 | GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE))((void)0); |
177 | |
178 | spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock); |
179 | |
180 | if (i915_is_ggtt(vm)((vm)->is_ggtt)) { |
181 | if (unlikely(overflows_type(vma->size, u32))__builtin_expect(!!((sizeof(vma->size) > sizeof(u32) && (vma->size) >> (8 * sizeof(u32)))), 0)) |
182 | goto err_unlock; |
183 | |
184 | vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, |
185 | i915_gem_object_get_tiling(obj), |
186 | i915_gem_object_get_stride(obj)); |
187 | if (unlikely(vma->fence_size < vma->size || /* overflow */__builtin_expect(!!(vma->fence_size < vma->size || vma ->fence_size > vm->total), 0) |
188 | vma->fence_size > vm->total)__builtin_expect(!!(vma->fence_size < vma->size || vma ->fence_size > vm->total), 0)) |
189 | goto err_unlock; |
190 | |
191 | GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT))((void)0); |
192 | |
193 | vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size, |
194 | i915_gem_object_get_tiling(obj), |
195 | i915_gem_object_get_stride(obj)); |
196 | GEM_BUG_ON(!is_power_of_2(vma->fence_alignment))((void)0); |
197 | |
198 | __set_bit(I915_VMA_GGTT_BIT14, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags)); |
199 | } |
200 | |
201 | rb = NULL((void *)0); |
202 | p = &obj->vma.tree.rb_node; |
203 | while (*p) { |
204 | long cmp; |
205 | |
206 | rb = *p; |
207 | pos = rb_entry(rb, struct i915_vma, obj_node)({ const __typeof( ((struct i915_vma *)0)->obj_node ) *__mptr = (rb); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof (struct i915_vma, obj_node) );}); |
208 | |
209 | /* |
210 | * If the view already exists in the tree, another thread |
211 | * already created a matching vma, so return the older instance |
212 | * and dispose of ours. |
213 | */ |
214 | cmp = i915_vma_compare(pos, vm, view); |
215 | if (cmp < 0) |
216 | p = &rb->rb_right__entry.rbe_right; |
217 | else if (cmp > 0) |
218 | p = &rb->rb_left__entry.rbe_left; |
219 | else |
220 | goto err_unlock; |
221 | } |
222 | rb_link_node(&vma->obj_node, rb, p); |
223 | rb_insert_color(&vma->obj_node, &obj->vma.tree)linux_root_RB_INSERT_COLOR((struct linux_root *)(&obj-> vma.tree), (&vma->obj_node)); |
224 | |
225 | if (i915_vma_is_ggtt(vma)) |
226 | /* |
227 | * We put the GGTT vma at the start of the vma-list, followed |
228 | * by the ppGGTT vma. This allows us to break early when |
229 | * iterating over only the GGTT vma for an object, see |
230 | * for_each_ggtt_vma() |
231 | */ |
232 | list_add(&vma->obj_link, &obj->vma.list); |
233 | else |
234 | list_add_tail(&vma->obj_link, &obj->vma.list); |
235 | |
236 | spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock); |
237 | |
238 | return vma; |
239 | |
240 | err_unlock: |
241 | spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock); |
242 | err_vma: |
243 | i915_vm_put(vm); |
244 | i915_vma_free(vma); |
245 | return pos; |
246 | } |
247 | |
248 | static struct i915_vma * |
249 | vma_lookup(struct drm_i915_gem_object *obj, |
250 | struct i915_address_space *vm, |
251 | const struct i915_ggtt_view *view) |
252 | { |
253 | struct rb_node *rb; |
254 | |
255 | rb = obj->vma.tree.rb_node; |
256 | while (rb) { |
257 | struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node)({ const __typeof( ((struct i915_vma *)0)->obj_node ) *__mptr = (rb); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof (struct i915_vma, obj_node) );}); |
258 | long cmp; |
259 | |
260 | cmp = i915_vma_compare(vma, vm, view); |
261 | if (cmp == 0) |
262 | return vma; |
263 | |
264 | if (cmp < 0) |
265 | rb = rb->rb_right__entry.rbe_right; |
266 | else |
267 | rb = rb->rb_left__entry.rbe_left; |
268 | } |
269 | |
270 | return NULL((void *)0); |
271 | } |
272 | |
273 | /** |
274 | * i915_vma_instance - return the singleton instance of the VMA |
275 | * @obj: parent &struct drm_i915_gem_object to be mapped |
276 | * @vm: address space in which the mapping is located |
277 | * @view: additional mapping requirements |
278 | * |
279 | * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with |
280 | * the same @view characteristics. If a match is not found, one is created. |
281 | * Once created, the VMA is kept until either the object is freed, or the |
282 | * address space is closed. |
283 | * |
284 | * Returns the vma, or an error pointer. |
285 | */ |
286 | struct i915_vma * |
287 | i915_vma_instance(struct drm_i915_gem_object *obj, |
288 | struct i915_address_space *vm, |
289 | const struct i915_ggtt_view *view) |
290 | { |
291 | struct i915_vma *vma; |
292 | |
293 | GEM_BUG_ON(view && !i915_is_ggtt(vm))((void)0); |
294 | GEM_BUG_ON(!atomic_read(&vm->open))((void)0); |
295 | |
296 | spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock); |
297 | vma = vma_lookup(obj, vm, view); |
298 | spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock); |
299 | |
300 | /* vma_create() will resolve the race if another creates the vma */ |
301 | if (unlikely(!vma)__builtin_expect(!!(!vma), 0)) |
302 | vma = vma_create(obj, vm, view); |
303 | |
304 | GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view))((void)0); |
305 | return vma; |
306 | } |
307 | |
308 | struct i915_vma_work { |
309 | struct dma_fence_work base; |
310 | struct i915_address_space *vm; |
311 | struct i915_vm_pt_stash stash; |
312 | struct i915_vma *vma; |
313 | struct drm_i915_gem_object *pinned; |
314 | struct i915_sw_dma_fence_cb cb; |
315 | enum i915_cache_level cache_level; |
316 | unsigned int flags; |
317 | }; |
318 | |
319 | static int __vma_bind(struct dma_fence_work *work) |
320 | { |
321 | struct i915_vma_work *vw = container_of(work, typeof(*vw), base)({ const __typeof( ((typeof(*vw) *)0)->base ) *__mptr = (work ); (typeof(*vw) *)( (char *)__mptr - __builtin_offsetof(typeof (*vw), base) );}); |
322 | struct i915_vma *vma = vw->vma; |
323 | |
324 | vma->ops->bind_vma(vw->vm, &vw->stash, |
325 | vma, vw->cache_level, vw->flags); |
326 | return 0; |
327 | } |
328 | |
329 | static void __vma_release(struct dma_fence_work *work) |
330 | { |
331 | struct i915_vma_work *vw = container_of(work, typeof(*vw), base)({ const __typeof( ((typeof(*vw) *)0)->base ) *__mptr = (work ); (typeof(*vw) *)( (char *)__mptr - __builtin_offsetof(typeof (*vw), base) );}); |
332 | |
333 | if (vw->pinned) { |
334 | __i915_gem_object_unpin_pages(vw->pinned); |
335 | i915_gem_object_put(vw->pinned); |
336 | } |
337 | |
338 | i915_vm_free_pt_stash(vw->vm, &vw->stash); |
339 | i915_vm_put(vw->vm); |
340 | } |
341 | |
342 | static const struct dma_fence_work_ops bind_ops = { |
343 | .name = "bind", |
344 | .work = __vma_bind, |
345 | .release = __vma_release, |
346 | }; |
347 | |
348 | struct i915_vma_work *i915_vma_work(void) |
349 | { |
350 | struct i915_vma_work *vw; |
351 | |
352 | vw = kzalloc(sizeof(*vw), GFP_KERNEL(0x0001 | 0x0004)); |
353 | if (!vw) |
354 | return NULL((void *)0); |
355 | |
356 | dma_fence_work_init(&vw->base, &bind_ops); |
357 | vw->base.dma.error = -EAGAIN35; /* disable the worker by default */ |
358 | |
359 | return vw; |
360 | } |
361 | |
362 | int i915_vma_wait_for_bind(struct i915_vma *vma) |
363 | { |
364 | int err = 0; |
365 | |
366 | if (rcu_access_pointer(vma->active.excl.fence)(vma->active.excl.fence)) { |
367 | struct dma_fence *fence; |
368 | |
369 | rcu_read_lock(); |
370 | fence = dma_fence_get_rcu_safe(&vma->active.excl.fence); |
371 | rcu_read_unlock(); |
372 | if (fence) { |
373 | err = dma_fence_wait(fence, MAX_SCHEDULE_TIMEOUT(0x7fffffff)); |
374 | dma_fence_put(fence); |
375 | } |
376 | } |
377 | |
378 | return err; |
379 | } |
380 | |
381 | /** |
382 | * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. |
383 | * @vma: VMA to map |
384 | * @cache_level: mapping cache level |
385 | * @flags: flags like global or local mapping |
386 | * @work: preallocated worker for allocating and binding the PTE |
387 | * |
388 | * DMA addresses are taken from the scatter-gather table of this object (or of |
389 | * this VMA in case of non-default GGTT views) and PTE entries set up. |
390 | * Note that DMA addresses are also the only part of the SG table we care about. |
391 | */ |
392 | int i915_vma_bind(struct i915_vma *vma, |
393 | enum i915_cache_level cache_level, |
394 | u32 flags, |
395 | struct i915_vma_work *work) |
396 | { |
397 | u32 bind_flags; |
398 | u32 vma_flags; |
399 | |
400 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0); |
401 | GEM_BUG_ON(vma->size > vma->node.size)((void)0); |
402 | |
403 | if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,({ ((void)0); 0; }) |
404 | vma->node.size,({ ((void)0); 0; }) |
405 | vma->vm->total))({ ((void)0); 0; })) |
406 | return -ENODEV19; |
407 | |
408 | if (GEM_DEBUG_WARN_ON(!flags)({ ((void)0); 0; })) |
409 | return -EINVAL22; |
410 | |
411 | bind_flags = flags; |
412 | bind_flags &= I915_VMA_GLOBAL_BIND((int)(1UL << (10))) | I915_VMA_LOCAL_BIND((int)(1UL << (11))); |
413 | |
414 | vma_flags = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*( &vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer (); __tmp; }); |
415 | vma_flags &= I915_VMA_GLOBAL_BIND((int)(1UL << (10))) | I915_VMA_LOCAL_BIND((int)(1UL << (11))); |
416 | |
417 | bind_flags &= ~vma_flags; |
418 | if (bind_flags == 0) |
419 | return 0; |
420 | |
421 | GEM_BUG_ON(!vma->pages)((void)0); |
422 | |
423 | trace_i915_vma_bind(vma, bind_flags); |
424 | if (work && bind_flags & vma->vm->bind_async_flags) { |
425 | struct dma_fence *prev; |
426 | |
427 | work->vma = vma; |
428 | work->cache_level = cache_level; |
429 | work->flags = bind_flags; |
430 | |
431 | /* |
432 | * Note we only want to chain up to the migration fence on |
433 | * the pages (not the object itself). As we don't track that, |
434 | * yet, we have to use the exclusive fence instead. |
435 | * |
436 | * Also note that we do not want to track the async vma as |
437 | * part of the obj->resv->excl_fence as it only affects |
438 | * execution and not content or object's backing store lifetime. |
439 | */ |
440 | prev = i915_active_set_exclusive(&vma->active, &work->base.dma); |
441 | if (prev) { |
442 | __i915_sw_fence_await_dma_fence(&work->base.chain, |
443 | prev, |
444 | &work->cb); |
445 | dma_fence_put(prev); |
446 | } |
447 | |
448 | work->base.dma.error = 0; /* enable the queue_work() */ |
449 | |
450 | if (vma->obj) { |
451 | __i915_gem_object_pin_pages(vma->obj); |
452 | work->pinned = i915_gem_object_get(vma->obj); |
453 | } |
454 | } else { |
455 | vma->ops->bind_vma(vma->vm, NULL((void *)0), vma, cache_level, bind_flags); |
456 | } |
457 | |
458 | atomic_or(bind_flags, &vma->flags)x86_atomic_setbits_u32(&vma->flags, bind_flags); |
459 | return 0; |
460 | } |
461 | |
462 | void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) |
463 | { |
464 | void __iomem *ptr; |
465 | int err; |
466 | |
467 | if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))({ __builtin_expect(!!(!!(!i915_vma_is_map_and_fenceable(vma) )), 0); })) { |
468 | err = -ENODEV19; |
469 | goto err; |
470 | } |
471 | |
472 | GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0); |
473 | GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))((void)0); |
474 | |
475 | ptr = READ_ONCE(vma->iomap)({ typeof(vma->iomap) __tmp = *(volatile typeof(vma->iomap ) *)&(vma->iomap); membar_datadep_consumer(); __tmp; } ); |
476 | if (ptr == NULL((void *)0)) { |
477 | #ifdef __linux__ |
478 | ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, |
479 | vma->node.start, |
480 | vma->node.size); |
481 | #else |
482 | struct drm_i915_privateinteldrm_softc *dev_priv = vma->vm->i915; |
483 | err = agp_map_subregion(dev_priv->agph, vma->node.start, |
484 | vma->node.size, &vma->bsh); |
485 | if (err) { |
486 | err = -err; |
487 | goto err; |
488 | } |
489 | ptr = bus_space_vaddr(dev_priv->bst, vma->bsh)((dev_priv->bst)->vaddr((vma->bsh))); |
490 | #endif |
491 | if (ptr == NULL((void *)0)) { |
492 | err = -ENOMEM12; |
493 | goto err; |
494 | } |
495 | |
496 | if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))__builtin_expect(!!(__sync_val_compare_and_swap(&vma-> iomap, ((void *)0), ptr)), 0)) { |
497 | #ifdef __linux__ |
498 | io_mapping_unmap(ptr); |
499 | #endif |
500 | ptr = vma->iomap; |
501 | } |
502 | } |
503 | |
504 | __i915_vma_pin(vma); |
505 | |
506 | err = i915_vma_pin_fence(vma); |
507 | if (err) |
508 | goto err_unpin; |
509 | |
510 | i915_vma_set_ggtt_write(vma); |
511 | |
512 | /* NB Access through the GTT requires the device to be awake. */ |
513 | return ptr; |
514 | |
515 | err_unpin: |
516 | __i915_vma_unpin(vma); |
517 | err: |
518 | return IO_ERR_PTR(err)((void *)ERR_PTR(err)); |
519 | } |
520 | |
521 | void i915_vma_flush_writes(struct i915_vma *vma) |
522 | { |
523 | if (i915_vma_unset_ggtt_write(vma)) |
524 | intel_gt_flush_ggtt_writes(vma->vm->gt); |
525 | } |
526 | |
527 | void i915_vma_unpin_iomap(struct i915_vma *vma) |
528 | { |
529 | GEM_BUG_ON(vma->iomap == NULL)((void)0); |
530 | |
531 | i915_vma_flush_writes(vma); |
532 | |
533 | i915_vma_unpin_fence(vma); |
534 | i915_vma_unpin(vma); |
535 | } |
536 | |
537 | void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags) |
538 | { |
539 | struct i915_vma *vma; |
540 | struct drm_i915_gem_object *obj; |
541 | |
542 | vma = fetch_and_zero(p_vma)({ typeof(*p_vma) __T = *(p_vma); *(p_vma) = (typeof(*p_vma)) 0; __T; }); |
543 | if (!vma) |
544 | return; |
545 | |
546 | obj = vma->obj; |
547 | GEM_BUG_ON(!obj)((void)0); |
548 | |
549 | i915_vma_unpin(vma); |
550 | |
551 | if (flags & I915_VMA_RELEASE_MAP(1UL << (0))) |
552 | i915_gem_object_unpin_map(obj); |
553 | |
554 | i915_gem_object_put(obj); |
555 | } |
556 | |
557 | bool_Bool i915_vma_misplaced(const struct i915_vma *vma, |
558 | u64 size, u64 alignment, u64 flags) |
559 | { |
560 | if (!drm_mm_node_allocated(&vma->node)) |
561 | return false0; |
562 | |
563 | if (test_bit(I915_VMA_ERROR_BIT13, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags))) |
564 | return true1; |
565 | |
566 | if (vma->node.size < size) |
567 | return true1; |
568 | |
569 | GEM_BUG_ON(alignment && !is_power_of_2(alignment))((void)0); |
570 | if (alignment && !IS_ALIGNED(vma->node.start, alignment)(((vma->node.start) & ((alignment) - 1)) == 0)) |
571 | return true1; |
572 | |
573 | if (flags & PIN_MAPPABLE(1ULL << (3)) && !i915_vma_is_map_and_fenceable(vma)) |
574 | return true1; |
575 | |
576 | if (flags & PIN_OFFSET_BIAS(1ULL << (6)) && |
577 | vma->node.start < (flags & PIN_OFFSET_MASK-(1ULL << (12)))) |
578 | return true1; |
579 | |
580 | if (flags & PIN_OFFSET_FIXED(1ULL << (7)) && |
581 | vma->node.start != (flags & PIN_OFFSET_MASK-(1ULL << (12)))) |
582 | return true1; |
583 | |
584 | return false0; |
585 | } |
586 | |
587 | void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) |
588 | { |
589 | bool_Bool mappable, fenceable; |
590 | |
591 | GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0); |
592 | GEM_BUG_ON(!vma->fence_size)((void)0); |
593 | |
594 | fenceable = (vma->node.size >= vma->fence_size && |
595 | IS_ALIGNED(vma->node.start, vma->fence_alignment)(((vma->node.start) & ((vma->fence_alignment) - 1)) == 0)); |
596 | |
597 | mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end; |
598 | |
599 | if (mappable && fenceable) |
600 | set_bit(I915_VMA_CAN_FENCE_BIT15, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags)); |
601 | else |
602 | clear_bit(I915_VMA_CAN_FENCE_BIT15, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags)); |
603 | } |
604 | |
605 | bool_Bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color) |
606 | { |
607 | struct drm_mm_node *node = &vma->node; |
608 | struct drm_mm_node *other; |
609 | |
610 | /* |
611 | * On some machines we have to be careful when putting differing types |
612 | * of snoopable memory together to avoid the prefetcher crossing memory |
613 | * domains and dying. During vm initialisation, we decide whether or not |
614 | * these constraints apply and set the drm_mm.color_adjust |
615 | * appropriately. |
616 | */ |
617 | if (!i915_vm_has_cache_coloring(vma->vm)) |
618 | return true1; |
619 | |
620 | /* Only valid to be called on an already inserted vma */ |
621 | GEM_BUG_ON(!drm_mm_node_allocated(node))((void)0); |
622 | GEM_BUG_ON(list_empty(&node->node_list))((void)0); |
623 | |
624 | other = list_prev_entry(node, node_list)({ const __typeof( ((typeof(*(node)) *)0)->node_list ) *__mptr = (((node)->node_list.prev)); (typeof(*(node)) *)( (char * )__mptr - __builtin_offsetof(typeof(*(node)), node_list) );}); |
625 | if (i915_node_color_differs(other, color) && |
626 | !drm_mm_hole_follows(other)) |
627 | return false0; |
628 | |
629 | other = list_next_entry(node, node_list)({ const __typeof( ((typeof(*(node)) *)0)->node_list ) *__mptr = (((node)->node_list.next)); (typeof(*(node)) *)( (char * )__mptr - __builtin_offsetof(typeof(*(node)), node_list) );}); |
630 | if (i915_node_color_differs(other, color) && |
631 | !drm_mm_hole_follows(node)) |
632 | return false0; |
633 | |
634 | return true1; |
635 | } |
636 | |
637 | /** |
638 | * i915_vma_insert - finds a slot for the vma in its address space |
639 | * @vma: the vma |
640 | * @size: requested size in bytes (can be larger than the VMA) |
641 | * @alignment: required alignment |
642 | * @flags: mask of PIN_* flags to use |
643 | * |
644 | * First we try to allocate some free space that meets the requirements for |
645 | * the VMA. Failiing that, if the flags permit, it will evict an old VMA, |
646 | * preferrably the oldest idle entry to make room for the new VMA. |
647 | * |
648 | * Returns: |
649 | * 0 on success, negative error code otherwise. |
650 | */ |
651 | static int |
652 | i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) |
653 | { |
654 | unsigned long color; |
655 | u64 start, end; |
656 | int ret; |
657 | |
658 | GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))((void)0); |
659 | GEM_BUG_ON(drm_mm_node_allocated(&vma->node))((void)0); |
660 | |
661 | size = max(size, vma->size)(((size)>(vma->size))?(size):(vma->size)); |
662 | alignment = max(alignment, vma->display_alignment)(((alignment)>(vma->display_alignment))?(alignment):(vma ->display_alignment)); |
663 | if (flags & PIN_MAPPABLE(1ULL << (3))) { |
664 | size = max_t(typeof(size), size, vma->fence_size)({ typeof(size) __max_a = (size); typeof(size) __max_b = (vma ->fence_size); __max_a > __max_b ? __max_a : __max_b; } ); |
665 | alignment = max_t(typeof(alignment),({ typeof(alignment) __max_a = (alignment); typeof(alignment) __max_b = (vma->fence_alignment); __max_a > __max_b ? __max_a : __max_b; }) |
666 | alignment, vma->fence_alignment)({ typeof(alignment) __max_a = (alignment); typeof(alignment) __max_b = (vma->fence_alignment); __max_a > __max_b ? __max_a : __max_b; }); |
667 | } |
668 | |
669 | GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE))((void)0); |
670 | GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT))((void)0); |
671 | GEM_BUG_ON(!is_power_of_2(alignment))((void)0); |
672 | |
673 | start = flags & PIN_OFFSET_BIAS(1ULL << (6)) ? flags & PIN_OFFSET_MASK-(1ULL << (12)) : 0; |
674 | GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE))((void)0); |
675 | |
676 | end = vma->vm->total; |
677 | if (flags & PIN_MAPPABLE(1ULL << (3))) |
678 | end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end)({ u64 __min_a = (end); u64 __min_b = (i915_vm_to_ggtt(vma-> vm)->mappable_end); __min_a < __min_b ? __min_a : __min_b ; }); |
679 | if (flags & PIN_ZONE_4G(1ULL << (4))) |
680 | end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE)({ u64 __min_a = (end); u64 __min_b = ((1ULL << 32) - ( 1ULL << (12))); __min_a < __min_b ? __min_a : __min_b ; }); |
681 | GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE))((void)0); |
682 | |
683 | /* If binding the object/GGTT view requires more space than the entire |
684 | * aperture has, reject it early before evicting everything in a vain |
685 | * attempt to find space. |
686 | */ |
687 | if (size > end) { |
688 | DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",__drm_dbg(DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n" , size, flags & (1ULL << (3)) ? "mappable" : "total" , end) |
689 | size, flags & PIN_MAPPABLE ? "mappable" : "total",__drm_dbg(DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n" , size, flags & (1ULL << (3)) ? "mappable" : "total" , end) |
690 | end)__drm_dbg(DRM_UT_CORE, "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n" , size, flags & (1ULL << (3)) ? "mappable" : "total" , end); |
691 | return -ENOSPC28; |
692 | } |
693 | |
694 | color = 0; |
695 | if (vma->obj && i915_vm_has_cache_coloring(vma->vm)) |
696 | color = vma->obj->cache_level; |
697 | |
698 | if (flags & PIN_OFFSET_FIXED(1ULL << (7))) { |
699 | u64 offset = flags & PIN_OFFSET_MASK-(1ULL << (12)); |
700 | if (!IS_ALIGNED(offset, alignment)(((offset) & ((alignment) - 1)) == 0) || |
701 | range_overflows(offset, size, end)({ typeof(offset) start__ = (offset); typeof(size) size__ = ( size); typeof(end) max__ = (end); (void)(&start__ == & size__); (void)(&start__ == &max__); start__ >= max__ || size__ > max__ - start__; })) |
702 | return -EINVAL22; |
703 | |
704 | ret = i915_gem_gtt_reserve(vma->vm, &vma->node, |
705 | size, offset, color, |
706 | flags); |
707 | if (ret) |
708 | return ret; |
709 | } else { |
710 | /* |
711 | * We only support huge gtt pages through the 48b PPGTT, |
712 | * however we also don't want to force any alignment for |
713 | * objects which need to be tightly packed into the low 32bits. |
714 | * |
715 | * Note that we assume that GGTT are limited to 4GiB for the |
716 | * forseeable future. See also i915_ggtt_offset(). |
717 | */ |
718 | if (upper_32_bits(end - 1)((u32)(((end - 1) >> 16) >> 16)) && |
719 | vma->page_sizes.sg > I915_GTT_PAGE_SIZE(1ULL << (12))) { |
720 | /* |
721 | * We can't mix 64K and 4K PTEs in the same page-table |
722 | * (2M block), and so to avoid the ugliness and |
723 | * complexity of coloring we opt for just aligning 64K |
724 | * objects to 2M. |
725 | */ |
726 | u64 page_alignment = |
727 | rounddown_pow_of_two(vma->page_sizes.sg | |
728 | I915_GTT_PAGE_SIZE_2M(1ULL << (21))); |
729 | |
730 | /* |
731 | * Check we don't expand for the limited Global GTT |
732 | * (mappable aperture is even more precious!). This |
733 | * also checks that we exclude the aliasing-ppgtt. |
734 | */ |
735 | GEM_BUG_ON(i915_vma_is_ggtt(vma))((void)0); |
736 | |
737 | alignment = max(alignment, page_alignment)(((alignment)>(page_alignment))?(alignment):(page_alignment )); |
738 | |
739 | if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K(1ULL << (16))) |
740 | size = round_up(size, I915_GTT_PAGE_SIZE_2M)((((size) + (((1ULL << (21))) - 1)) / ((1ULL << ( 21)))) * ((1ULL << (21)))); |
741 | } |
742 | |
743 | ret = i915_gem_gtt_insert(vma->vm, &vma->node, |
744 | size, alignment, color, |
745 | start, end, flags); |
746 | if (ret) |
747 | return ret; |
748 | |
749 | GEM_BUG_ON(vma->node.start < start)((void)0); |
750 | GEM_BUG_ON(vma->node.start + vma->node.size > end)((void)0); |
751 | } |
752 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0); |
753 | GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color))((void)0); |
754 | |
755 | list_add_tail(&vma->vm_link, &vma->vm->bound_list); |
756 | |
757 | return 0; |
758 | } |
759 | |
760 | static void |
761 | i915_vma_detach(struct i915_vma *vma) |
762 | { |
763 | GEM_BUG_ON(!drm_mm_node_allocated(&vma->node))((void)0); |
764 | GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND))((void)0); |
765 | |
766 | /* |
767 | * And finally now the object is completely decoupled from this |
768 | * vma, we can drop its hold on the backing storage and allow |
769 | * it to be reaped by the shrinker. |
770 | */ |
771 | list_del(&vma->vm_link); |
772 | } |
773 | |
774 | static bool_Bool try_qad_pin(struct i915_vma *vma, unsigned int flags) |
775 | { |
776 | unsigned int bound; |
777 | bool_Bool pinned = true1; |
778 | |
779 | bound = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*( &vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer (); __tmp; }); |
780 | do { |
781 | if (unlikely(flags & ~bound)__builtin_expect(!!(flags & ~bound), 0)) |
782 | return false0; |
783 | |
784 | if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))__builtin_expect(!!(bound & (0x200 | ((int)(1UL << ( 13))))), 0)) |
785 | return false0; |
786 | |
787 | if (!(bound & I915_VMA_PIN_MASK0x3ff)) |
788 | goto unpinned; |
789 | |
790 | GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0)((void)0); |
791 | } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); |
792 | |
793 | return true1; |
794 | |
795 | unpinned: |
796 | /* |
797 | * If pin_count==0, but we are bound, check under the lock to avoid |
798 | * racing with a concurrent i915_vma_unbind(). |
799 | */ |
800 | mutex_lock(&vma->vm->mutex)rw_enter_write(&vma->vm->mutex); |
801 | do { |
802 | if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))__builtin_expect(!!(bound & (0x200 | ((int)(1UL << ( 13))))), 0)) { |
803 | pinned = false0; |
804 | break; |
805 | } |
806 | |
807 | if (unlikely(flags & ~bound)__builtin_expect(!!(flags & ~bound), 0)) { |
808 | pinned = false0; |
809 | break; |
810 | } |
811 | } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); |
812 | mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex); |
813 | |
814 | return pinned; |
815 | } |
816 | |
817 | static int vma_get_pages(struct i915_vma *vma) |
818 | { |
819 | int err = 0; |
820 | |
821 | if (atomic_add_unless(&vma->pages_count, 1, 0)) |
822 | return 0; |
823 | |
824 | /* Allocations ahoy! */ |
825 | if (mutex_lock_interruptible(&vma->pages_mutex)) |
826 | return -EINTR4; |
827 | |
828 | if (!atomic_read(&vma->pages_count)({ typeof(*(&vma->pages_count)) __tmp = *(volatile typeof (*(&vma->pages_count)) *)&(*(&vma->pages_count )); membar_datadep_consumer(); __tmp; })) { |
829 | if (vma->obj) { |
830 | err = i915_gem_object_pin_pages(vma->obj); |
831 | if (err) |
832 | goto unlock; |
833 | } |
834 | |
835 | err = vma->ops->set_pages(vma); |
836 | if (err) { |
837 | if (vma->obj) |
838 | i915_gem_object_unpin_pages(vma->obj); |
839 | goto unlock; |
840 | } |
841 | } |
842 | atomic_inc(&vma->pages_count)__sync_fetch_and_add(&vma->pages_count, 1); |
843 | |
844 | unlock: |
845 | mutex_unlock(&vma->pages_mutex)rw_exit_write(&vma->pages_mutex); |
846 | |
847 | return err; |
848 | } |
849 | |
850 | static void __vma_put_pages(struct i915_vma *vma, unsigned int count) |
851 | { |
852 | /* We allocate under vma_get_pages, so beware the shrinker */ |
853 | mutex_lock_nested(&vma->pages_mutex, SINGLE_DEPTH_NESTING)rw_enter_write(&vma->pages_mutex); |
854 | GEM_BUG_ON(atomic_read(&vma->pages_count) < count)((void)0); |
855 | if (atomic_sub_return(count, &vma->pages_count)__sync_sub_and_fetch(&vma->pages_count, count) == 0) { |
856 | vma->ops->clear_pages(vma); |
857 | GEM_BUG_ON(vma->pages)((void)0); |
858 | if (vma->obj) |
859 | i915_gem_object_unpin_pages(vma->obj); |
860 | } |
861 | mutex_unlock(&vma->pages_mutex)rw_exit_write(&vma->pages_mutex); |
862 | } |
863 | |
864 | static void vma_put_pages(struct i915_vma *vma) |
865 | { |
866 | if (atomic_add_unless(&vma->pages_count, -1, 1)) |
867 | return; |
868 | |
869 | __vma_put_pages(vma, 1); |
870 | } |
871 | |
872 | static void vma_unbind_pages(struct i915_vma *vma) |
873 | { |
874 | unsigned int count; |
875 | |
876 | lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0); |
877 | |
878 | /* The upper portion of pages_count is the number of bindings */ |
879 | count = atomic_read(&vma->pages_count)({ typeof(*(&vma->pages_count)) __tmp = *(volatile typeof (*(&vma->pages_count)) *)&(*(&vma->pages_count )); membar_datadep_consumer(); __tmp; }); |
880 | count >>= I915_VMA_PAGES_BIAS24; |
881 | GEM_BUG_ON(!count)((void)0); |
882 | |
883 | __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS24); |
884 | } |
885 | |
886 | int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, |
887 | u64 size, u64 alignment, u64 flags) |
888 | { |
889 | struct i915_vma_work *work = NULL((void *)0); |
890 | intel_wakeref_t wakeref = 0; |
891 | unsigned int bound; |
892 | int err; |
893 | |
894 | #ifdef CONFIG_PROVE_LOCKING |
895 | if (debug_locks && lockdep_is_held(&vma->vm->i915->drm.struct_mutex)) |
896 | WARN_ON(!ww)({ int __ret = !!((!ww)); if (__ret) printf("%s", "WARN_ON(" "!ww" ")"); __builtin_expect(!!(__ret), 0); }); |
897 | #endif |
898 | |
899 | BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND)extern char _ctassert[(!((1ULL << (10)) != ((int)(1UL << (10))))) ? 1 : -1 ] __attribute__((__unused__)); |
900 | BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND)extern char _ctassert[(!((1ULL << (11)) != ((int)(1UL << (11))))) ? 1 : -1 ] __attribute__((__unused__)); |
901 | |
902 | GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)))((void)0); |
903 | |
904 | /* First try and grab the pin without rebinding the vma */ |
905 | if (try_qad_pin(vma, flags & I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) |
906 | return 0; |
907 | |
908 | err = vma_get_pages(vma); |
909 | if (err) |
910 | return err; |
911 | |
912 | if (flags & PIN_GLOBAL(1ULL << (10))) |
913 | wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm); |
914 | |
915 | if (flags & vma->vm->bind_async_flags) { |
916 | work = i915_vma_work(); |
917 | if (!work) { |
918 | err = -ENOMEM12; |
919 | goto err_rpm; |
920 | } |
921 | |
922 | work->vm = i915_vm_get(vma->vm); |
923 | |
924 | /* Allocate enough page directories to used PTE */ |
925 | if (vma->vm->allocate_va_range) { |
926 | err = i915_vm_alloc_pt_stash(vma->vm, |
927 | &work->stash, |
928 | vma->size); |
929 | if (err) |
930 | goto err_fence; |
931 | |
932 | err = i915_vm_pin_pt_stash(vma->vm, |
933 | &work->stash); |
934 | if (err) |
935 | goto err_fence; |
936 | } |
937 | } |
938 | |
939 | /* |
940 | * Differentiate between user/kernel vma inside the aliasing-ppgtt. |
941 | * |
942 | * We conflate the Global GTT with the user's vma when using the |
943 | * aliasing-ppgtt, but it is still vitally important to try and |
944 | * keep the use cases distinct. For example, userptr objects are |
945 | * not allowed inside the Global GTT as that will cause lock |
946 | * inversions when we have to evict them the mmu_notifier callbacks - |
947 | * but they are allowed to be part of the user ppGTT which can never |
948 | * be mapped. As such we try to give the distinct users of the same |
949 | * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt |
950 | * and i915_ppgtt separate]. |
951 | * |
952 | * NB this may cause us to mask real lock inversions -- while the |
953 | * code is safe today, lockdep may not be able to spot future |
954 | * transgressions. |
955 | */ |
956 | err = mutex_lock_interruptible_nested(&vma->vm->mutex,mutex_lock_interruptible(&vma->vm->mutex) |
957 | !(flags & PIN_GLOBAL))mutex_lock_interruptible(&vma->vm->mutex); |
958 | if (err) |
959 | goto err_fence; |
960 | |
961 | /* No more allocations allowed now we hold vm->mutex */ |
962 | |
963 | if (unlikely(i915_vma_is_closed(vma))__builtin_expect(!!(i915_vma_is_closed(vma)), 0)) { |
964 | err = -ENOENT2; |
965 | goto err_unlock; |
966 | } |
967 | |
968 | bound = atomic_read(&vma->flags)({ typeof(*(&vma->flags)) __tmp = *(volatile typeof(*( &vma->flags)) *)&(*(&vma->flags)); membar_datadep_consumer (); __tmp; }); |
969 | if (unlikely(bound & I915_VMA_ERROR)__builtin_expect(!!(bound & ((int)(1UL << (13)))), 0 )) { |
970 | err = -ENOMEM12; |
971 | goto err_unlock; |
972 | } |
973 | |
974 | if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))__builtin_expect(!!(!((bound + 1) & 0x3ff)), 0)) { |
975 | err = -EAGAIN35; /* pins are meant to be fairly temporary */ |
976 | goto err_unlock; |
977 | } |
978 | |
979 | if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))__builtin_expect(!!(!(flags & ~bound & (((int)(1UL << (10))) | ((int)(1UL << (11)))))), 0)) { |
980 | __i915_vma_pin(vma); |
981 | goto err_unlock; |
982 | } |
983 | |
984 | err = i915_active_acquire(&vma->active); |
985 | if (err) |
986 | goto err_unlock; |
987 | |
988 | if (!(bound & I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) { |
989 | err = i915_vma_insert(vma, size, alignment, flags); |
990 | if (err) |
991 | goto err_active; |
992 | |
993 | if (i915_is_ggtt(vma->vm)((vma->vm)->is_ggtt)) |
994 | __i915_vma_set_map_and_fenceable(vma); |
995 | } |
996 | |
997 | GEM_BUG_ON(!vma->pages)((void)0); |
998 | err = i915_vma_bind(vma, |
999 | vma->obj ? vma->obj->cache_level : 0, |
1000 | flags, work); |
1001 | if (err) |
1002 | goto err_remove; |
1003 | |
1004 | /* There should only be at most 2 active bindings (user, global) */ |
1005 | GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound)((void)0); |
1006 | atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count)__sync_fetch_and_add(&vma->pages_count, ((1UL << (24)) | 1)); |
1007 | list_move_tail(&vma->vm_link, &vma->vm->bound_list); |
1008 | |
1009 | __i915_vma_pin(vma); |
1010 | GEM_BUG_ON(!i915_vma_is_pinned(vma))((void)0); |
1011 | GEM_BUG_ON(!i915_vma_is_bound(vma, flags))((void)0); |
1012 | GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags))((void)0); |
1013 | |
1014 | err_remove: |
1015 | if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) { |
1016 | i915_vma_detach(vma); |
1017 | drm_mm_remove_node(&vma->node); |
1018 | } |
1019 | err_active: |
1020 | i915_active_release(&vma->active); |
1021 | err_unlock: |
1022 | mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex); |
1023 | err_fence: |
1024 | if (work) |
1025 | dma_fence_work_commit_imm(&work->base); |
1026 | err_rpm: |
1027 | if (wakeref) |
1028 | intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref); |
1029 | vma_put_pages(vma); |
1030 | return err; |
1031 | } |
1032 | |
1033 | static void flush_idle_contexts(struct intel_gt *gt) |
1034 | { |
1035 | struct intel_engine_cs *engine; |
1036 | enum intel_engine_id id; |
1037 | |
1038 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else |
1039 | intel_engine_flush_barriers(engine); |
1040 | |
1041 | intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT(0x7fffffff)); |
1042 | } |
1043 | |
1044 | int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, |
1045 | u32 align, unsigned int flags) |
1046 | { |
1047 | struct i915_address_space *vm = vma->vm; |
1048 | int err; |
1049 | |
1050 | GEM_BUG_ON(!i915_vma_is_ggtt(vma))((void)0); |
1051 | |
1052 | do { |
1053 | err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL(1ULL << (10))); |
1054 | if (err != -ENOSPC28) { |
1055 | if (!err) { |
1056 | err = i915_vma_wait_for_bind(vma); |
1057 | if (err) |
1058 | i915_vma_unpin(vma); |
1059 | } |
1060 | return err; |
1061 | } |
1062 | |
1063 | /* Unlike i915_vma_pin, we don't take no for an answer! */ |
1064 | flush_idle_contexts(vm->gt); |
1065 | if (mutex_lock_interruptible(&vm->mutex) == 0) { |
1066 | i915_gem_evict_vm(vm); |
1067 | mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex); |
1068 | } |
1069 | } while (1); |
1070 | } |
1071 | |
1072 | static void __vma_close(struct i915_vma *vma, struct intel_gt *gt) |
1073 | { |
1074 | /* |
1075 | * We defer actually closing, unbinding and destroying the VMA until |
1076 | * the next idle point, or if the object is freed in the meantime. By |
1077 | * postponing the unbind, we allow for it to be resurrected by the |
1078 | * client, avoiding the work required to rebind the VMA. This is |
1079 | * advantageous for DRI, where the client/server pass objects |
1080 | * between themselves, temporarily opening a local VMA to the |
1081 | * object, and then closing it again. The same object is then reused |
1082 | * on the next frame (or two, depending on the depth of the swap queue) |
1083 | * causing us to rebind the VMA once more. This ends up being a lot |
1084 | * of wasted work for the steady state. |
1085 | */ |
1086 | GEM_BUG_ON(i915_vma_is_closed(vma))((void)0); |
1087 | list_add(&vma->closed_link, >->closed_vma); |
1088 | } |
1089 | |
1090 | void i915_vma_close(struct i915_vma *vma) |
1091 | { |
1092 | struct intel_gt *gt = vma->vm->gt; |
1093 | unsigned long flags; |
1094 | |
1095 | if (i915_vma_is_ggtt(vma)) |
1096 | return; |
1097 | |
1098 | GEM_BUG_ON(!atomic_read(&vma->open_count))((void)0); |
1099 | if (atomic_dec_and_lock_irqsave(&vma->open_count,atomic_dec_and_lock(&vma->open_count, >->closed_lock ) |
1100 | >->closed_lock,atomic_dec_and_lock(&vma->open_count, >->closed_lock ) |
1101 | flags)atomic_dec_and_lock(&vma->open_count, >->closed_lock )) { |
1102 | __vma_close(vma, gt); |
1103 | spin_unlock_irqrestore(>->closed_lock, flags)do { (void)(flags); mtx_leave(>->closed_lock); } while (0); |
1104 | } |
1105 | } |
1106 | |
1107 | static void __i915_vma_remove_closed(struct i915_vma *vma) |
1108 | { |
1109 | struct intel_gt *gt = vma->vm->gt; |
1110 | |
1111 | spin_lock_irq(>->closed_lock)mtx_enter(>->closed_lock); |
1112 | list_del_init(&vma->closed_link); |
1113 | spin_unlock_irq(>->closed_lock)mtx_leave(>->closed_lock); |
1114 | } |
1115 | |
1116 | void i915_vma_reopen(struct i915_vma *vma) |
1117 | { |
1118 | if (i915_vma_is_closed(vma)) |
1119 | __i915_vma_remove_closed(vma); |
1120 | } |
1121 | |
1122 | void i915_vma_release(struct kref *ref) |
1123 | { |
1124 | struct i915_vma *vma = container_of(ref, typeof(*vma), ref)({ const __typeof( ((typeof(*vma) *)0)->ref ) *__mptr = (ref ); (typeof(*vma) *)( (char *)__mptr - __builtin_offsetof(typeof (*vma), ref) );}); |
1125 | |
1126 | if (drm_mm_node_allocated(&vma->node)) { |
1127 | mutex_lock(&vma->vm->mutex)rw_enter_write(&vma->vm->mutex); |
1128 | atomic_and(~I915_VMA_PIN_MASK, &vma->flags)__sync_fetch_and_and(&vma->flags, ~0x3ff); |
1129 | WARN_ON(__i915_vma_unbind(vma))({ int __ret = !!((__i915_vma_unbind(vma))); if (__ret) printf ("%s", "WARN_ON(" "__i915_vma_unbind(vma)" ")"); __builtin_expect (!!(__ret), 0); }); |
1130 | mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex); |
1131 | GEM_BUG_ON(drm_mm_node_allocated(&vma->node))((void)0); |
1132 | } |
1133 | GEM_BUG_ON(i915_vma_is_active(vma))((void)0); |
1134 | |
1135 | if (vma->obj) { |
1136 | struct drm_i915_gem_object *obj = vma->obj; |
1137 | |
1138 | spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock); |
1139 | list_del(&vma->obj_link); |
1140 | if (!RB_EMPTY_NODE(&vma->obj_node)((&vma->obj_node)->__entry.rbe_parent == &vma-> obj_node)) |
1141 | rb_erase(&vma->obj_node, &obj->vma.tree)linux_root_RB_REMOVE((struct linux_root *)(&obj->vma.tree ), (&vma->obj_node)); |
1142 | spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock); |
1143 | } |
1144 | |
1145 | __i915_vma_remove_closed(vma); |
1146 | i915_vm_put(vma->vm); |
1147 | |
1148 | i915_active_fini(&vma->active); |
1149 | i915_vma_free(vma); |
1150 | } |
1151 | |
1152 | void i915_vma_parked(struct intel_gt *gt) |
1153 | { |
1154 | struct i915_vma *vma, *next; |
1155 | DRM_LIST_HEAD(closed)struct list_head closed = { &(closed), &(closed) }; |
1156 | |
1157 | spin_lock_irq(>->closed_lock)mtx_enter(>->closed_lock); |
1158 | list_for_each_entry_safe(vma, next, >->closed_vma, closed_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link ) *__mptr = ((>->closed_vma)->next); (__typeof(* vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link ) );}), next = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link ) *__mptr = (vma->closed_link.next); (__typeof(*vma) *)( ( char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link ) );}); &vma->closed_link != (>->closed_vma); vma = next, next = ({ const __typeof( ((__typeof(*next) *)0) ->closed_link ) *__mptr = (next->closed_link.next); (__typeof (*next) *)( (char *)__mptr - __builtin_offsetof(__typeof(*next ), closed_link) );})) { |
1159 | struct drm_i915_gem_object *obj = vma->obj; |
1160 | struct i915_address_space *vm = vma->vm; |
1161 | |
1162 | /* XXX All to avoid keeping a reference on i915_vma itself */ |
1163 | |
1164 | if (!kref_get_unless_zero(&obj->base.refcount)) |
1165 | continue; |
1166 | |
1167 | if (!i915_vm_tryopen(vm)) { |
1168 | i915_gem_object_put(obj); |
1169 | continue; |
1170 | } |
1171 | |
1172 | list_move(&vma->closed_link, &closed); |
1173 | } |
1174 | spin_unlock_irq(>->closed_lock)mtx_leave(>->closed_lock); |
1175 | |
1176 | /* As the GT is held idle, no vma can be reopened as we destroy them */ |
1177 | list_for_each_entry_safe(vma, next, &closed, closed_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link ) *__mptr = ((&closed)->next); (__typeof(*vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link) ) ;}), next = ({ const __typeof( ((__typeof(*vma) *)0)->closed_link ) *__mptr = (vma->closed_link.next); (__typeof(*vma) *)( ( char *)__mptr - __builtin_offsetof(__typeof(*vma), closed_link ) );}); &vma->closed_link != (&closed); vma = next , next = ({ const __typeof( ((__typeof(*next) *)0)->closed_link ) *__mptr = (next->closed_link.next); (__typeof(*next) *) ( (char *)__mptr - __builtin_offsetof(__typeof(*next), closed_link ) );})) { |
1178 | struct drm_i915_gem_object *obj = vma->obj; |
1179 | struct i915_address_space *vm = vma->vm; |
1180 | |
1181 | INIT_LIST_HEAD(&vma->closed_link); |
1182 | __i915_vma_put(vma); |
1183 | |
1184 | i915_gem_object_put(obj); |
1185 | i915_vm_close(vm); |
1186 | } |
1187 | } |
1188 | |
1189 | static void __i915_vma_iounmap(struct i915_vma *vma) |
1190 | { |
1191 | GEM_BUG_ON(i915_vma_is_pinned(vma))((void)0); |
1192 | |
1193 | if (vma->iomap == NULL((void *)0)) |
1194 | return; |
1195 | |
1196 | #ifdef __linux__ |
1197 | io_mapping_unmap(vma->iomap); |
1198 | #else |
1199 | struct drm_i915_privateinteldrm_softc *dev_priv = vma->vm->i915; |
1200 | agp_unmap_subregion(dev_priv->agph, vma->bsh, vma->node.size); |
1201 | #endif |
1202 | vma->iomap = NULL((void *)0); |
1203 | } |
1204 | |
1205 | void i915_vma_revoke_mmap(struct i915_vma *vma) |
1206 | { |
1207 | struct drm_vma_offset_node *node; |
1208 | u64 vma_offset; |
1209 | |
1210 | if (!i915_vma_has_userfault(vma)) |
1211 | return; |
1212 | |
1213 | GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma))((void)0); |
1214 | GEM_BUG_ON(!vma->obj->userfault_count)((void)0); |
1215 | |
1216 | node = &vma->mmo->vma_node; |
1217 | vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT12; |
Value stored to 'vma_offset' is never read | |
1218 | #ifdef __linux__ |
1219 | unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, |
1220 | drm_vma_node_offset_addr(node) + vma_offset, |
1221 | vma->size, |
1222 | 1); |
1223 | #else |
1224 | struct drm_i915_privateinteldrm_softc *dev_priv = vma->obj->base.dev->dev_private; |
1225 | struct vm_page *pg; |
1226 | |
1227 | for (pg = &dev_priv->pgs[atop(vma->node.start)((vma->node.start) >> 12)]; |
1228 | pg != &dev_priv->pgs[atop(vma->node.start + vma->size)((vma->node.start + vma->size) >> 12)]; |
1229 | pg++) |
1230 | pmap_page_protect(pg, PROT_NONE0x00); |
1231 | #endif |
1232 | |
1233 | i915_vma_unset_userfault(vma); |
1234 | if (!--vma->obj->userfault_count) |
1235 | list_del(&vma->obj->userfault_link); |
1236 | } |
1237 | |
1238 | static int |
1239 | __i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma) |
1240 | { |
1241 | return __i915_request_await_exclusive(rq, &vma->active); |
1242 | } |
1243 | |
1244 | int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) |
1245 | { |
1246 | int err; |
1247 | |
1248 | GEM_BUG_ON(!i915_vma_is_pinned(vma))((void)0); |
1249 | |
1250 | /* Wait for the vma to be bound before we start! */ |
1251 | err = __i915_request_await_bind(rq, vma); |
1252 | if (err) |
1253 | return err; |
1254 | |
1255 | return i915_active_add_request(&vma->active, rq); |
1256 | } |
1257 | |
1258 | int i915_vma_move_to_active(struct i915_vma *vma, |
1259 | struct i915_request *rq, |
1260 | unsigned int flags) |
1261 | { |
1262 | struct drm_i915_gem_object *obj = vma->obj; |
1263 | int err; |
1264 | |
1265 | assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while (0); |
1266 | |
1267 | err = __i915_vma_move_to_active(vma, rq); |
1268 | if (unlikely(err)__builtin_expect(!!(err), 0)) |
1269 | return err; |
1270 | |
1271 | if (flags & EXEC_OBJECT_WRITE(1<<2)) { |
1272 | struct intel_frontbuffer *front; |
1273 | |
1274 | front = __intel_frontbuffer_get(obj); |
1275 | if (unlikely(front)__builtin_expect(!!(front), 0)) { |
1276 | if (intel_frontbuffer_invalidate(front, ORIGIN_CS)) |
1277 | i915_active_add_request(&front->write, rq); |
1278 | intel_frontbuffer_put(front); |
1279 | } |
1280 | |
1281 | dma_resv_add_excl_fence(vma->resv, &rq->fence); |
1282 | obj->write_domain = I915_GEM_DOMAIN_RENDER0x00000002; |
1283 | obj->read_domains = 0; |
1284 | } else { |
1285 | err = dma_resv_reserve_shared(vma->resv, 1); |
1286 | if (unlikely(err)__builtin_expect(!!(err), 0)) |
1287 | return err; |
1288 | |
1289 | dma_resv_add_shared_fence(vma->resv, &rq->fence); |
1290 | obj->write_domain = 0; |
1291 | } |
1292 | |
1293 | if (flags & EXEC_OBJECT_NEEDS_FENCE(1<<0) && vma->fence) |
1294 | i915_active_add_request(&vma->fence->active, rq); |
1295 | |
1296 | obj->read_domains |= I915_GEM_GPU_DOMAINS(0x00000002 | 0x00000004 | 0x00000008 | 0x00000010 | 0x00000020 ); |
1297 | obj->mm.dirty = true1; |
1298 | |
1299 | GEM_BUG_ON(!i915_vma_is_active(vma))((void)0); |
1300 | return 0; |
1301 | } |
1302 | |
1303 | void __i915_vma_evict(struct i915_vma *vma) |
1304 | { |
1305 | GEM_BUG_ON(i915_vma_is_pinned(vma))((void)0); |
1306 | |
1307 | if (i915_vma_is_map_and_fenceable(vma)) { |
1308 | /* Force a pagefault for domain tracking on next user access */ |
1309 | i915_vma_revoke_mmap(vma); |
1310 | |
1311 | /* |
1312 | * Check that we have flushed all writes through the GGTT |
1313 | * before the unbind, other due to non-strict nature of those |
1314 | * indirect writes they may end up referencing the GGTT PTE |
1315 | * after the unbind. |
1316 | * |
1317 | * Note that we may be concurrently poking at the GGTT_WRITE |
1318 | * bit from set-domain, as we mark all GGTT vma associated |
1319 | * with an object. We know this is for another vma, as we |
1320 | * are currently unbinding this one -- so if this vma will be |
1321 | * reused, it will be refaulted and have its dirty bit set |
1322 | * before the next write. |
1323 | */ |
1324 | i915_vma_flush_writes(vma); |
1325 | |
1326 | /* release the fence reg _after_ flushing */ |
1327 | i915_vma_revoke_fence(vma); |
1328 | |
1329 | __i915_vma_iounmap(vma); |
1330 | clear_bit(I915_VMA_CAN_FENCE_BIT15, __i915_vma_flags(vma)((unsigned long *)&(vma)->flags)); |
1331 | } |
1332 | GEM_BUG_ON(vma->fence)((void)0); |
1333 | GEM_BUG_ON(i915_vma_has_userfault(vma))((void)0); |
1334 | |
1335 | if (likely(atomic_read(&vma->vm->open))__builtin_expect(!!(({ typeof(*(&vma->vm->open)) __tmp = *(volatile typeof(*(&vma->vm->open)) *)&(*(& vma->vm->open)); membar_datadep_consumer(); __tmp; })), 1)) { |
1336 | trace_i915_vma_unbind(vma); |
1337 | vma->ops->unbind_vma(vma->vm, vma); |
1338 | } |
1339 | atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),__sync_fetch_and_and(&vma->flags, ~((((int)(1UL << (10))) | ((int)(1UL << (11)))) | ((int)(1UL << ( 13))) | ((int)(1UL << (17))))) |
1340 | &vma->flags)__sync_fetch_and_and(&vma->flags, ~((((int)(1UL << (10))) | ((int)(1UL << (11)))) | ((int)(1UL << ( 13))) | ((int)(1UL << (17))))); |
1341 | |
1342 | i915_vma_detach(vma); |
1343 | vma_unbind_pages(vma); |
1344 | } |
1345 | |
1346 | int __i915_vma_unbind(struct i915_vma *vma) |
1347 | { |
1348 | int ret; |
1349 | |
1350 | lockdep_assert_held(&vma->vm->mutex)do { (void)(&vma->vm->mutex); } while(0); |
1351 | |
1352 | if (!drm_mm_node_allocated(&vma->node)) |
1353 | return 0; |
1354 | |
1355 | if (i915_vma_is_pinned(vma)) { |
1356 | vma_print_allocator(vma, "is pinned"); |
1357 | return -EAGAIN35; |
1358 | } |
1359 | |
1360 | /* |
1361 | * After confirming that no one else is pinning this vma, wait for |
1362 | * any laggards who may have crept in during the wait (through |
1363 | * a residual pin skipping the vm->mutex) to complete. |
1364 | */ |
1365 | ret = i915_vma_sync(vma); |
1366 | if (ret) |
1367 | return ret; |
1368 | |
1369 | GEM_BUG_ON(i915_vma_is_active(vma))((void)0); |
1370 | __i915_vma_evict(vma); |
1371 | |
1372 | drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ |
1373 | return 0; |
1374 | } |
1375 | |
1376 | int i915_vma_unbind(struct i915_vma *vma) |
1377 | { |
1378 | struct i915_address_space *vm = vma->vm; |
1379 | intel_wakeref_t wakeref = 0; |
1380 | int err; |
1381 | |
1382 | /* Optimistic wait before taking the mutex */ |
1383 | err = i915_vma_sync(vma); |
1384 | if (err) |
1385 | return err; |
1386 | |
1387 | if (!drm_mm_node_allocated(&vma->node)) |
1388 | return 0; |
1389 | |
1390 | if (i915_vma_is_pinned(vma)) { |
1391 | vma_print_allocator(vma, "is pinned"); |
1392 | return -EAGAIN35; |
1393 | } |
1394 | |
1395 | if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND((int)(1UL << (10))))) |
1396 | /* XXX not always required: nop_clear_range */ |
1397 | wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); |
1398 | |
1399 | err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref)mutex_lock_interruptible(&vma->vm->mutex); |
1400 | if (err) |
1401 | goto out_rpm; |
1402 | |
1403 | err = __i915_vma_unbind(vma); |
1404 | mutex_unlock(&vm->mutex)rw_exit_write(&vm->mutex); |
1405 | |
1406 | out_rpm: |
1407 | if (wakeref) |
1408 | intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref); |
1409 | return err; |
1410 | } |
1411 | |
1412 | struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma) |
1413 | { |
1414 | i915_gem_object_make_unshrinkable(vma->obj); |
1415 | return vma; |
1416 | } |
1417 | |
1418 | void i915_vma_make_shrinkable(struct i915_vma *vma) |
1419 | { |
1420 | i915_gem_object_make_shrinkable(vma->obj); |
1421 | } |
1422 | |
1423 | void i915_vma_make_purgeable(struct i915_vma *vma) |
1424 | { |
1425 | i915_gem_object_make_purgeable(vma->obj); |
1426 | } |
1427 | |
1428 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0 |
1429 | #include "selftests/i915_vma.c" |
1430 | #endif |
1431 | |
1432 | static void i915_global_vma_shrink(void) |
1433 | { |
1434 | #ifdef notyet |
1435 | kmem_cache_shrink(global.slab_vmas); |
1436 | #endif |
1437 | } |
1438 | |
1439 | static void i915_global_vma_exit(void) |
1440 | { |
1441 | #ifdef __linux__ |
1442 | kmem_cache_destroy(global.slab_vmas); |
1443 | #else |
1444 | pool_destroy(&global.slab_vmas); |
1445 | #endif |
1446 | } |
1447 | |
1448 | static struct i915_global_vma global = { { |
1449 | .shrink = i915_global_vma_shrink, |
1450 | .exit = i915_global_vma_exit, |
1451 | } }; |
1452 | |
1453 | int __init i915_global_vma_init(void) |
1454 | { |
1455 | #ifdef __linux__ |
1456 | global.slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); |
1457 | if (!global.slab_vmas) |
1458 | return -ENOMEM12; |
1459 | #else |
1460 | pool_init(&global.slab_vmas, sizeof(struct i915_vma), |
1461 | CACHELINESIZE64, IPL_NONE0x0, 0, "drmvma", NULL((void *)0)); |
1462 | #endif |
1463 | |
1464 | i915_global_register(&global.base); |
1465 | return 0; |
1466 | } |