Bug Summary

File:dev/pci/drm/ttm/ttm_bo.c
Warning:line 1103, column 7
Dereference of null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ttm_bo.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/ttm/ttm_bo.c
1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2/**************************************************************************
3 *
4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28/*
29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30 */
31
32#define pr_fmt(fmt)"[TTM] " fmt "[TTM] " fmt
33
34#include <drm/ttm/ttm_bo_driver.h>
35#include <drm/ttm/ttm_placement.h>
36#include <linux/jiffies.h>
37#include <linux/slab.h>
38#include <linux/sched.h>
39#include <linux/mm.h>
40#include <linux/file.h>
41#include <linux/module.h>
42#include <linux/atomic.h>
43#include <linux/dma-resv.h>
44
45#include "ttm_module.h"
46
47static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
48 struct ttm_placement *placement)
49{
50 struct drm_printer p = drm_debug_printer(TTM_PFX"[TTM] ");
51 struct ttm_resource_manager *man;
52 int i, mem_type;
53
54 drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n",
55 bo, bo->resource->num_pages, bo->base.size >> 10,
56 bo->base.size >> 20);
57 for (i = 0; i < placement->num_placement; i++) {
58 mem_type = placement->placement[i].mem_type;
59 drm_printf(&p, " placement[%d]=0x%08X (%d)\n",
60 i, placement->placement[i].flags, mem_type);
61 man = ttm_manager_type(bo->bdev, mem_type);
62 ttm_resource_manager_debug(man, &p);
63 }
64}
65
66/**
67 * ttm_bo_move_to_lru_tail
68 *
69 * @bo: The buffer object.
70 *
71 * Move this BO to the tail of all lru lists used to lookup and reserve an
72 * object. This function must be called with struct ttm_global::lru_lock
73 * held, and is used to make a BO less likely to be considered for eviction.
74 */
75void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
76{
77 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
78
79 if (bo->resource)
80 ttm_resource_move_to_lru_tail(bo->resource);
81}
82EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
83
84/**
85 * ttm_bo_set_bulk_move - update BOs bulk move object
86 *
87 * @bo: The buffer object.
88 *
89 * Update the BOs bulk move object, making sure that resources are added/removed
90 * as well. A bulk move allows to move many resource on the LRU at once,
91 * resulting in much less overhead of maintaining the LRU.
92 * The only requirement is that the resources stay together on the LRU and are
93 * never separated. This is enforces by setting the bulk_move structure on a BO.
94 * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
95 * their LRU list.
96 */
97void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
98 struct ttm_lru_bulk_move *bulk)
99{
100 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
101
102 if (bo->bulk_move == bulk)
103 return;
104
105 spin_lock(&bo->bdev->lru_lock)mtx_enter(&bo->bdev->lru_lock);
106 if (bo->resource)
107 ttm_resource_del_bulk_move(bo->resource, bo);
108 bo->bulk_move = bulk;
109 if (bo->resource)
110 ttm_resource_add_bulk_move(bo->resource, bo);
111 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
112}
113EXPORT_SYMBOL(ttm_bo_set_bulk_move);
114
115static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
116 struct ttm_resource *mem, bool_Bool evict,
117 struct ttm_operation_ctx *ctx,
118 struct ttm_place *hop)
119{
120 struct ttm_device *bdev = bo->bdev;
121 bool_Bool old_use_tt, new_use_tt;
122 int ret;
123
124 old_use_tt = bo->resource &&
14
←
Assuming field 'resource' is null
→
125 ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
126 new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
127
128 ttm_bo_unmap_virtual(bo);
15
←
Calling 'ttm_bo_unmap_virtual'
→
129
130 /*
131 * Create and bind a ttm if required.
132 */
133
134 if (new_use_tt) {
135 /* Zero init the new TTM structure if the old location should
136 * have used one as well.
137 */
138 ret = ttm_tt_create(bo, old_use_tt);
139 if (ret)
140 goto out_err;
141
142 if (mem->mem_type != TTM_PL_SYSTEM0) {
143 ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
144 if (ret)
145 goto out_err;
146 }
147 }
148
149 ret = dma_resv_reserve_fences(bo->base.resv, 1);
150 if (ret)
151 goto out_err;
152
153 ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
154 if (ret) {
155 if (ret == -EMULTIHOP82)
156 return ret;
157 goto out_err;
158 }
159
160 ctx->bytes_moved += bo->base.size;
161 return 0;
162
163out_err:
164 if (!old_use_tt)
165 ttm_bo_tt_destroy(bo);
166
167 return ret;
168}
169
170/*
171 * Call bo::reserved.
172 * Will release GPU memory type usage on destruction.
173 * This is the place to put in driver specific hooks to release
174 * driver private resources.
175 * Will release the bo::reserved lock.
176 */
177
178static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
179{
180 if (bo->bdev->funcs->delete_mem_notify)
181 bo->bdev->funcs->delete_mem_notify(bo);
182
183 ttm_bo_tt_destroy(bo);
184 ttm_resource_free(bo, &bo->resource);
185}
186
187static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
188{
189 int r;
190
191 if (bo->base.resv == &bo->base._resv)
192 return 0;
193
194 BUG_ON(!dma_resv_trylock(&bo->base._resv))((!(!dma_resv_trylock(&bo->base._resv))) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/dev/pci/drm/ttm/ttm_bo.c", 194,
"!(!dma_resv_trylock(&bo->base._resv))"));
195
196 r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
197 dma_resv_unlock(&bo->base._resv);
198 if (r)
199 return r;
200
201 if (bo->type != ttm_bo_type_sg) {
202 /* This works because the BO is about to be destroyed and nobody
203 * reference it any more. The only tricky case is the trylock on
204 * the resv object while holding the lru_lock.
205 */
206 spin_lock(&bo->bdev->lru_lock)mtx_enter(&bo->bdev->lru_lock);
207 bo->base.resv = &bo->base._resv;
208 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
209 }
210
211 return r;
212}
213
214static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
215{
216 struct dma_resv *resv = &bo->base._resv;
217 struct dma_resv_iter cursor;
218 struct dma_fence *fence;
219
220 dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
221 dma_resv_for_each_fence_unlocked(&cursor, fence)for (fence = dma_resv_iter_first_unlocked(&cursor); fence
; fence = dma_resv_iter_next_unlocked(&cursor)) {
222 if (!fence->ops->signaled)
223 dma_fence_enable_sw_signaling(fence);
224 }
225 dma_resv_iter_end(&cursor);
226}
227
228/**
229 * ttm_bo_cleanup_refs
230 * If bo idle, remove from lru lists, and unref.
231 * If not idle, block if possible.
232 *
233 * Must be called with lru_lock and reservation held, this function
234 * will drop the lru lock and optionally the reservation lock before returning.
235 *
236 * @bo: The buffer object to clean-up
237 * @interruptible: Any sleeps should occur interruptibly.
238 * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead.
239 * @unlock_resv: Unlock the reservation lock as well.
240 */
241
242static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
243 bool_Bool interruptible, bool_Bool no_wait_gpu,
244 bool_Bool unlock_resv)
245{
246 struct dma_resv *resv = &bo->base._resv;
247 int ret;
248
249 if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
250 ret = 0;
251 else
252 ret = -EBUSY16;
253
254 if (ret && !no_wait_gpu) {
255 long lret;
256
257 if (unlock_resv)
258 dma_resv_unlock(bo->base.resv);
259 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
260
261 lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
262 interruptible,
263 30 * HZhz);
264
265 if (lret < 0)
266 return lret;
267 else if (lret == 0)
268 return -EBUSY16;
269
270 spin_lock(&bo->bdev->lru_lock)mtx_enter(&bo->bdev->lru_lock);
271 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
272 /*
273 * We raced, and lost, someone else holds the reservation now,
274 * and is probably busy in ttm_bo_cleanup_memtype_use.
275 *
276 * Even if it's not the case, because we finished waiting any
277 * delayed destruction would succeed, so just return success
278 * here.
279 */
280 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
281 return 0;
282 }
283 ret = 0;
284 }
285
286 if (ret || unlikely(list_empty(&bo->ddestroy))__builtin_expect(!!(list_empty(&bo->ddestroy)), 0)) {
287 if (unlock_resv)
288 dma_resv_unlock(bo->base.resv);
289 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
290 return ret;
291 }
292
293 list_del_init(&bo->ddestroy);
294 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
295 ttm_bo_cleanup_memtype_use(bo);
296
297 if (unlock_resv)
298 dma_resv_unlock(bo->base.resv);
299
300 ttm_bo_put(bo);
301
302 return 0;
303}
304
305/*
306 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
307 * encountered buffers.
308 */
309bool_Bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool_Bool remove_all)
310{
311 struct list_head removed;
312 bool_Bool empty;
313
314 INIT_LIST_HEAD(&removed);
315
316 spin_lock(&bdev->lru_lock)mtx_enter(&bdev->lru_lock);
317 while (!list_empty(&bdev->ddestroy)) {
318 struct ttm_buffer_object *bo;
319
320 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,({ const __typeof( ((struct ttm_buffer_object *)0)->ddestroy
) *__mptr = ((&bdev->ddestroy)->next); (struct ttm_buffer_object
*)( (char *)__mptr - __builtin_offsetof(struct ttm_buffer_object
, ddestroy) );})
321 ddestroy)({ const __typeof( ((struct ttm_buffer_object *)0)->ddestroy
) *__mptr = ((&bdev->ddestroy)->next); (struct ttm_buffer_object
*)( (char *)__mptr - __builtin_offsetof(struct ttm_buffer_object
, ddestroy) );});
322 list_move_tail(&bo->ddestroy, &removed);
323 if (!ttm_bo_get_unless_zero(bo))
324 continue;
325
326 if (remove_all || bo->base.resv != &bo->base._resv) {
327 spin_unlock(&bdev->lru_lock)mtx_leave(&bdev->lru_lock);
328 dma_resv_lock(bo->base.resv, NULL((void *)0));
329
330 spin_lock(&bdev->lru_lock)mtx_enter(&bdev->lru_lock);
331 ttm_bo_cleanup_refs(bo, false0, !remove_all, true1);
332
333 } else if (dma_resv_trylock(bo->base.resv)) {
334 ttm_bo_cleanup_refs(bo, false0, !remove_all, true1);
335 } else {
336 spin_unlock(&bdev->lru_lock)mtx_leave(&bdev->lru_lock);
337 }
338
339 ttm_bo_put(bo);
340 spin_lock(&bdev->lru_lock)mtx_enter(&bdev->lru_lock);
341 }
342 list_splice_tail(&removed, &bdev->ddestroy);
343 empty = list_empty(&bdev->ddestroy);
344 spin_unlock(&bdev->lru_lock)mtx_leave(&bdev->lru_lock);
345
346 return empty;
347}
348
349static void ttm_bo_release(struct kref *kref)
350{
351 struct ttm_buffer_object *bo =
352 container_of(kref, struct ttm_buffer_object, kref)({ const __typeof( ((struct ttm_buffer_object *)0)->kref )
*__mptr = (kref); (struct ttm_buffer_object *)( (char *)__mptr
- __builtin_offsetof(struct ttm_buffer_object, kref) );});
353 struct ttm_device *bdev = bo->bdev;
354 int ret;
355
356 WARN_ON_ONCE(bo->pin_count)({ static int __warned; int __ret = !!(bo->pin_count); if (
__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "bo->pin_count", "/usr/src/sys/dev/pci/drm/ttm/ttm_bo.c"
, 356); __warned = 1; } __builtin_expect(!!(__ret), 0); });
357 WARN_ON_ONCE(bo->bulk_move)({ static int __warned; int __ret = !!(bo->bulk_move); if (
__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "bo->bulk_move", "/usr/src/sys/dev/pci/drm/ttm/ttm_bo.c"
, 357); __warned = 1; } __builtin_expect(!!(__ret), 0); });
358
359 if (!bo->deleted) {
360 ret = ttm_bo_individualize_resv(bo);
361 if (ret) {
362 /* Last resort, if we fail to allocate memory for the
363 * fences block for the BO to become idle
364 */
365 dma_resv_wait_timeout(bo->base.resv,
366 DMA_RESV_USAGE_BOOKKEEP, false0,
367 30 * HZhz);
368 }
369
370 if (bo->bdev->funcs->release_notify)
371 bo->bdev->funcs->release_notify(bo);
372
373 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
374 ttm_mem_io_free(bdev, bo->resource);
375 }
376
377 if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP) ||
378 !dma_resv_trylock(bo->base.resv)) {
379 /* The BO is not idle, resurrect it for delayed destroy */
380 ttm_bo_flush_all_fences(bo);
381 bo->deleted = true1;
382
383 spin_lock(&bo->bdev->lru_lock)mtx_enter(&bo->bdev->lru_lock);
384
385 /*
386 * Make pinned bos immediately available to
387 * shrinkers, now that they are queued for
388 * destruction.
389 *
390 * FIXME: QXL is triggering this. Can be removed when the
391 * driver is fixed.
392 */
393 if (bo->pin_count) {
394 bo->pin_count = 0;
395 ttm_resource_move_to_lru_tail(bo->resource);
396 }
397
398 kref_init(&bo->kref);
399 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
400 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
401
402 schedule_delayed_work(&bdev->wq,
403 ((HZhz / 100) < 1) ? 1 : HZhz / 100);
404 return;
405 }
406
407 spin_lock(&bo->bdev->lru_lock)mtx_enter(&bo->bdev->lru_lock);
408 list_del(&bo->ddestroy);
409 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
410
411 ttm_bo_cleanup_memtype_use(bo);
412 dma_resv_unlock(bo->base.resv);
413
414 atomic_dec(&ttm_glob.bo_count)__sync_fetch_and_sub(&ttm_glob.bo_count, 1);
415 bo->destroy(bo);
416}
417
418void ttm_bo_put(struct ttm_buffer_object *bo)
419{
420 kref_put(&bo->kref, ttm_bo_release);
421}
422EXPORT_SYMBOL(ttm_bo_put);
423
424int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev)
425{
426 return cancel_delayed_work_sync(&bdev->wq);
427}
428EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
429
430void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched)
431{
432 if (resched)
433 schedule_delayed_work(&bdev->wq,
434 ((HZhz / 100) < 1) ? 1 : HZhz / 100);
435}
436EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
437
438static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
439 struct ttm_resource **mem,
440 struct ttm_operation_ctx *ctx,
441 struct ttm_place *hop)
442{
443 struct ttm_placement hop_placement;
444 struct ttm_resource *hop_mem;
445 int ret;
446
447 hop_placement.num_placement = hop_placement.num_busy_placement = 1;
448 hop_placement.placement = hop_placement.busy_placement = hop;
449
450 /* find space in the bounce domain */
451 ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
452 if (ret)
453 return ret;
454 /* move to the bounce domain */
455 ret = ttm_bo_handle_move_mem(bo, hop_mem, false0, ctx, NULL((void *)0));
456 if (ret) {
457 ttm_resource_free(bo, &hop_mem);
458 return ret;
459 }
460 return 0;
461}
462
463static int ttm_bo_evict(struct ttm_buffer_object *bo,
464 struct ttm_operation_ctx *ctx)
465{
466 struct ttm_device *bdev = bo->bdev;
467 struct ttm_resource *evict_mem;
468 struct ttm_placement placement;
469 struct ttm_place hop;
470 int ret = 0;
471
472 memset(&hop, 0, sizeof(hop))__builtin_memset((&hop), (0), (sizeof(hop)));
473
474 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
475
476 placement.num_placement = 0;
477 placement.num_busy_placement = 0;
478 bdev->funcs->evict_flags(bo, &placement);
479
480 if (!placement.num_placement && !placement.num_busy_placement) {
481 ret = ttm_bo_wait(bo, true1, false0);
482 if (ret)
483 return ret;
484
485 /*
486 * Since we've already synced, this frees backing store
487 * immediately.
488 */
489 return ttm_bo_pipeline_gutting(bo);
490 }
491
492 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
493 if (ret) {
494 if (ret != -ERESTARTSYS4) {
495 pr_err("Failed to find memory space for buffer 0x%p eviction\n",printk("\0013" "[TTM] " "Failed to find memory space for buffer 0x%p eviction\n"
, bo)
496 bo)printk("\0013" "[TTM] " "Failed to find memory space for buffer 0x%p eviction\n"
, bo);
497 ttm_bo_mem_space_debug(bo, &placement);
498 }
499 goto out;
500 }
501
502 do {
503 ret = ttm_bo_handle_move_mem(bo, evict_mem, true1, ctx, &hop);
504 if (ret != -EMULTIHOP82)
505 break;
506
507 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
508 } while (!ret);
509
510 if (ret) {
511 ttm_resource_free(bo, &evict_mem);
512 if (ret != -ERESTARTSYS4 && ret != -EINTR4)
513 pr_err("Buffer eviction failed\n")printk("\0013" "[TTM] " "Buffer eviction failed\n");
514 }
515out:
516 return ret;
517}
518
519bool_Bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
520 const struct ttm_place *place)
521{
522 struct ttm_resource *res = bo->resource;
523 struct ttm_device *bdev = bo->bdev;
524
525 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
526 if (bo->resource->mem_type == TTM_PL_SYSTEM0)
527 return true1;
528
529 /* Don't evict this BO if it's outside of the
530 * requested placement range
531 */
532 return ttm_resource_intersects(bdev, res, place, bo->base.size);
533}
534EXPORT_SYMBOL(ttm_bo_eviction_valuable);
535
536/*
537 * Check the target bo is allowable to be evicted or swapout, including cases:
538 *
539 * a. if share same reservation object with ctx->resv, have assumption
540 * reservation objects should already be locked, so not lock again and
541 * return true directly when either the opreation allow_reserved_eviction
542 * or the target bo already is in delayed free list;
543 *
544 * b. Otherwise, trylock it.
545 */
546static bool_Bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
547 struct ttm_operation_ctx *ctx,
548 const struct ttm_place *place,
549 bool_Bool *locked, bool_Bool *busy)
550{
551 bool_Bool ret = false0;
552
553 if (bo->pin_count) {
554 *locked = false0;
555 if (busy)
556 *busy = false0;
557 return false0;
558 }
559
560 if (bo->base.resv == ctx->resv) {
561 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
562 if (ctx->allow_res_evict)
563 ret = true1;
564 *locked = false0;
565 if (busy)
566 *busy = false0;
567 } else {
568 ret = dma_resv_trylock(bo->base.resv);
569 *locked = ret;
570 if (busy)
571 *busy = !ret;
572 }
573
574 if (ret && place && (bo->resource->mem_type != place->mem_type ||
575 !bo->bdev->funcs->eviction_valuable(bo, place))) {
576 ret = false0;
577 if (*locked) {
578 dma_resv_unlock(bo->base.resv);
579 *locked = false0;
580 }
581 }
582
583 return ret;
584}
585
586/**
587 * ttm_mem_evict_wait_busy - wait for a busy BO to become available
588 *
589 * @busy_bo: BO which couldn't be locked with trylock
590 * @ctx: operation context
591 * @ticket: acquire ticket
592 *
593 * Try to lock a busy buffer object to avoid failing eviction.
594 */
595static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
596 struct ttm_operation_ctx *ctx,
597 struct ww_acquire_ctx *ticket)
598{
599 int r;
600
601 if (!busy_bo || !ticket)
602 return -EBUSY16;
603
604 if (ctx->interruptible)
605 r = dma_resv_lock_interruptible(busy_bo->base.resv,
606 ticket);
607 else
608 r = dma_resv_lock(busy_bo->base.resv, ticket);
609
610 /*
611 * TODO: It would be better to keep the BO locked until allocation is at
612 * least tried one more time, but that would mean a much larger rework
613 * of TTM.
614 */
615 if (!r)
616 dma_resv_unlock(busy_bo->base.resv);
617
618 return r == -EDEADLK11 ? -EBUSY16 : r;
619}
620
621int ttm_mem_evict_first(struct ttm_device *bdev,
622 struct ttm_resource_manager *man,
623 const struct ttm_place *place,
624 struct ttm_operation_ctx *ctx,
625 struct ww_acquire_ctx *ticket)
626{
627 struct ttm_buffer_object *bo = NULL((void *)0), *busy_bo = NULL((void *)0);
628 struct ttm_resource_cursor cursor;
629 struct ttm_resource *res;
630 bool_Bool locked = false0;
631 int ret;
632
633 spin_lock(&bdev->lru_lock)mtx_enter(&bdev->lru_lock);
634 ttm_resource_manager_for_each_res(man, &cursor, res)for (res = ttm_resource_manager_first(man, &cursor); res;
res = ttm_resource_manager_next(man, &cursor, res)) {
635 bool_Bool busy;
636
637 if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
638 &locked, &busy)) {
639 if (busy && !busy_bo && ticket !=
640 dma_resv_locking_ctx(res->bo->base.resv))
641 busy_bo = res->bo;
642 continue;
643 }
644
645 if (ttm_bo_get_unless_zero(res->bo)) {
646 bo = res->bo;
647 break;
648 }
649 if (locked)
650 dma_resv_unlock(res->bo->base.resv);
651 }
652
653 if (!bo) {
654 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
655 busy_bo = NULL((void *)0);
656 spin_unlock(&bdev->lru_lock)mtx_leave(&bdev->lru_lock);
657 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
658 if (busy_bo)
659 ttm_bo_put(busy_bo);
660 return ret;
661 }
662
663 if (bo->deleted) {
664 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
665 ctx->no_wait_gpu, locked);
666 ttm_bo_put(bo);
667 return ret;
668 }
669
670 spin_unlock(&bdev->lru_lock)mtx_leave(&bdev->lru_lock);
671
672 ret = ttm_bo_evict(bo, ctx);
673 if (locked)
674 ttm_bo_unreserve(bo);
675 else
676 ttm_bo_move_to_lru_tail_unlocked(bo);
677
678 ttm_bo_put(bo);
679 return ret;
680}
681
682/**
683 * ttm_bo_pin - Pin the buffer object.
684 * @bo: The buffer object to pin
685 *
686 * Make sure the buffer is not evicted any more during memory pressure.
687 * @bo must be unpinned again by calling ttm_bo_unpin().
688 */
689void ttm_bo_pin(struct ttm_buffer_object *bo)
690{
691 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
692 WARN_ON_ONCE(!kref_read(&bo->kref))({ static int __warned; int __ret = !!(!kref_read(&bo->
kref)); if (__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "!kref_read(&bo->kref)", "/usr/src/sys/dev/pci/drm/ttm/ttm_bo.c"
, 692); __warned = 1; } __builtin_expect(!!(__ret), 0); });
693 spin_lock(&bo->bdev->lru_lock)mtx_enter(&bo->bdev->lru_lock);
694 if (bo->resource)
695 ttm_resource_del_bulk_move(bo->resource, bo);
696 ++bo->pin_count;
697 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
698}
699EXPORT_SYMBOL(ttm_bo_pin);
700
701/**
702 * ttm_bo_unpin - Unpin the buffer object.
703 * @bo: The buffer object to unpin
704 *
705 * Allows the buffer object to be evicted again during memory pressure.
706 */
707void ttm_bo_unpin(struct ttm_buffer_object *bo)
708{
709 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
710 WARN_ON_ONCE(!kref_read(&bo->kref))({ static int __warned; int __ret = !!(!kref_read(&bo->
kref)); if (__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "!kref_read(&bo->kref)", "/usr/src/sys/dev/pci/drm/ttm/ttm_bo.c"
, 710); __warned = 1; } __builtin_expect(!!(__ret), 0); });
711 if (WARN_ON_ONCE(!bo->pin_count)({ static int __warned; int __ret = !!(!bo->pin_count); if
(__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "!bo->pin_count", "/usr/src/sys/dev/pci/drm/ttm/ttm_bo.c"
, 711); __warned = 1; } __builtin_expect(!!(__ret), 0); }))
712 return;
713
714 spin_lock(&bo->bdev->lru_lock)mtx_enter(&bo->bdev->lru_lock);
715 --bo->pin_count;
716 if (bo->resource)
717 ttm_resource_add_bulk_move(bo->resource, bo);
718 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
719}
720EXPORT_SYMBOL(ttm_bo_unpin);
721
722/*
723 * Add the last move fence to the BO as kernel dependency and reserve a new
724 * fence slot.
725 */
726static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
727 struct ttm_resource_manager *man,
728 struct ttm_resource *mem,
729 bool_Bool no_wait_gpu)
730{
731 struct dma_fence *fence;
732 int ret;
733
734 spin_lock(&man->move_lock)mtx_enter(&man->move_lock);
735 fence = dma_fence_get(man->move);
736 spin_unlock(&man->move_lock)mtx_leave(&man->move_lock);
737
738 if (!fence)
739 return 0;
740
741 if (no_wait_gpu) {
742 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY16;
743 dma_fence_put(fence);
744 return ret;
745 }
746
747 dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
748
749 ret = dma_resv_reserve_fences(bo->base.resv, 1);
750 dma_fence_put(fence);
751 return ret;
752}
753
754/*
755 * Repeatedly evict memory from the LRU for @mem_type until we create enough
756 * space, or we've evicted everything and there isn't enough space.
757 */
758static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
759 const struct ttm_place *place,
760 struct ttm_resource **mem,
761 struct ttm_operation_ctx *ctx)
762{
763 struct ttm_device *bdev = bo->bdev;
764 struct ttm_resource_manager *man;
765 struct ww_acquire_ctx *ticket;
766 int ret;
767
768 man = ttm_manager_type(bdev, place->mem_type);
769 ticket = dma_resv_locking_ctx(bo->base.resv);
770 do {
771 ret = ttm_resource_alloc(bo, place, mem);
772 if (likely(!ret)__builtin_expect(!!(!ret), 1))
773 break;
774 if (unlikely(ret != -ENOSPC)__builtin_expect(!!(ret != -28), 0))
775 return ret;
776 ret = ttm_mem_evict_first(bdev, man, place, ctx,
777 ticket);
778 if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0))
779 return ret;
780 } while (1);
781
782 return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
783}
784
785/*
786 * Creates space for memory region @mem according to its type.
787 *
788 * This function first searches for free space in compatible memory types in
789 * the priority order defined by the driver. If free space isn't found, then
790 * ttm_bo_mem_force_space is attempted in priority order to evict and find
791 * space.
792 */
793int ttm_bo_mem_space(struct ttm_buffer_object *bo,
794 struct ttm_placement *placement,
795 struct ttm_resource **mem,
796 struct ttm_operation_ctx *ctx)
797{
798 struct ttm_device *bdev = bo->bdev;
799 bool_Bool type_found = false0;
800 int i, ret;
801
802 ret = dma_resv_reserve_fences(bo->base.resv, 1);
803 if (unlikely(ret)__builtin_expect(!!(ret), 0))
804 return ret;
805
806 for (i = 0; i < placement->num_placement; ++i) {
807 const struct ttm_place *place = &placement->placement[i];
808 struct ttm_resource_manager *man;
809
810 man = ttm_manager_type(bdev, place->mem_type);
811 if (!man || !ttm_resource_manager_used(man))
812 continue;
813
814 type_found = true1;
815 ret = ttm_resource_alloc(bo, place, mem);
816 if (ret == -ENOSPC28)
817 continue;
818 if (unlikely(ret)__builtin_expect(!!(ret), 0))
819 goto error;
820
821 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
822 if (unlikely(ret)__builtin_expect(!!(ret), 0)) {
823 ttm_resource_free(bo, mem);
824 if (ret == -EBUSY16)
825 continue;
826
827 goto error;
828 }
829 return 0;
830 }
831
832 for (i = 0; i < placement->num_busy_placement; ++i) {
833 const struct ttm_place *place = &placement->busy_placement[i];
834 struct ttm_resource_manager *man;
835
836 man = ttm_manager_type(bdev, place->mem_type);
837 if (!man || !ttm_resource_manager_used(man))
838 continue;
839
840 type_found = true1;
841 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
842 if (likely(!ret)__builtin_expect(!!(!ret), 1))
843 return 0;
844
845 if (ret && ret != -EBUSY16)
846 goto error;
847 }
848
849 ret = -ENOMEM12;
850 if (!type_found) {
851 pr_err(TTM_PFX "No compatible memory type found\n")printk("\0013" "[TTM] " "[TTM] " "No compatible memory type found\n"
);
852 ret = -EINVAL22;
853 }
854
855error:
856 return ret;
857}
858EXPORT_SYMBOL(ttm_bo_mem_space);
859
860static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
861 struct ttm_placement *placement,
862 struct ttm_operation_ctx *ctx)
863{
864 struct ttm_resource *mem;
865 struct ttm_place hop;
866 int ret;
867
868 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
869
870 /*
871 * Determine where to move the buffer.
872 *
873 * If driver determines move is going to need
874 * an extra step then it will return -EMULTIHOP
875 * and the buffer will be moved to the temporary
876 * stop and the driver will be called to make
877 * the second hop.
878 */
879 ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
880 if (ret)
881 return ret;
882bounce:
883 ret = ttm_bo_handle_move_mem(bo, mem, false0, ctx, &hop);
884 if (ret == -EMULTIHOP82) {
885 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
886 if (ret)
887 goto out;
888 /* try and move to final place now. */
889 goto bounce;
890 }
891out:
892 if (ret)
893 ttm_resource_free(bo, &mem);
894 return ret;
895}
896
897int ttm_bo_validate(struct ttm_buffer_object *bo,
898 struct ttm_placement *placement,
899 struct ttm_operation_ctx *ctx)
900{
901 int ret;
902
903 dma_resv_assert_held(bo->base.resv)do { (void)(&(bo->base.resv)->lock.base); } while(0
);
904
905 /*
906 * Remove the backing store if no placement is given.
907 */
908 if (!placement->num_placement && !placement->num_busy_placement)
909 return ttm_bo_pipeline_gutting(bo);
910
911 /*
912 * Check whether we need to move buffer.
913 */
914 if (!bo->resource || !ttm_resource_compat(bo->resource, placement)) {
915 ret = ttm_bo_move_buffer(bo, placement, ctx);
916 if (ret)
917 return ret;
918 }
919 /*
920 * We might need to add a TTM.
921 */
922 if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM0) {
923 ret = ttm_tt_create(bo, true1);
924 if (ret)
925 return ret;
926 }
927 return 0;
928}
929EXPORT_SYMBOL(ttm_bo_validate);
930
931/**
932 * ttm_bo_init_reserved
933 *
934 * @bdev: Pointer to a ttm_device struct.
935 * @bo: Pointer to a ttm_buffer_object to be initialized.
936 * @type: Requested type of buffer object.
937 * @placement: Initial placement for buffer object.
938 * @alignment: Data alignment in pages.
939 * @ctx: TTM operation context for memory allocation.
940 * @sg: Scatter-gather table.
941 * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
942 * @destroy: Destroy function. Use NULL for kfree().
943 *
944 * This function initializes a pre-allocated struct ttm_buffer_object.
945 * As this object may be part of a larger structure, this function,
946 * together with the @destroy function, enables driver-specific objects
947 * derived from a ttm_buffer_object.
948 *
949 * On successful return, the caller owns an object kref to @bo. The kref and
950 * list_kref are usually set to 1, but note that in some situations, other
951 * tasks may already be holding references to @bo as well.
952 * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
953 * and it is the caller's responsibility to call ttm_bo_unreserve.
954 *
955 * If a failure occurs, the function will call the @destroy function. Thus,
956 * after a failure, dereferencing @bo is illegal and will likely cause memory
957 * corruption.
958 *
959 * Returns
960 * -ENOMEM: Out of memory.
961 * -EINVAL: Invalid placement flags.
962 * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
963 */
964int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
965 enum ttm_bo_type type, struct ttm_placement *placement,
966 uint32_t alignment, struct ttm_operation_ctx *ctx,
967 struct sg_table *sg, struct dma_resv *resv,
968 void (*destroy) (struct ttm_buffer_object *))
969{
970 static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM0 };
971 int ret;
972
973 kref_init(&bo->kref);
974 INIT_LIST_HEAD(&bo->ddestroy);
975 bo->bdev = bdev;
976 bo->type = type;
977 bo->page_alignment = alignment;
978 bo->destroy = destroy;
979 bo->pin_count = 0;
980 bo->sg = sg;
981 bo->bulk_move = NULL((void *)0);
982 if (resv)
983 bo->base.resv = resv;
984 else
985 bo->base.resv = &bo->base._resv;
986 atomic_inc(&ttm_glob.bo_count)__sync_fetch_and_add(&ttm_glob.bo_count, 1);
987
988 ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource);
989 if (unlikely(ret)__builtin_expect(!!(ret), 0)) {
990 ttm_bo_put(bo);
991 return ret;
992 }
993
994 /*
995 * For ttm_bo_type_device buffers, allocate
996 * address space from the device.
997 */
998 if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) {
999 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1000 PFN_UP(bo->base.size)(((bo->base.size) + (1 << 12)-1) >> 12));
1001 if (ret)
1002 goto err_put;
1003 }
1004
1005 /* passed reservation objects should already be locked,
1006 * since otherwise lockdep will be angered in radeon.
1007 */
1008 if (!resv)
1009 WARN_ON(!dma_resv_trylock(bo->base.resv))({ int __ret = !!(!dma_resv_trylock(bo->base.resv)); if (__ret
) printf("WARNING %s failed at %s:%d\n", "!dma_resv_trylock(bo->base.resv)"
, "/usr/src/sys/dev/pci/drm/ttm/ttm_bo.c", 1009); __builtin_expect
(!!(__ret), 0); });
1010 else
1011 dma_resv_assert_held(resv)do { (void)(&(resv)->lock.base); } while(0);
1012
1013 ret = ttm_bo_validate(bo, placement, ctx);
1014 if (unlikely(ret)__builtin_expect(!!(ret), 0))
1015 goto err_unlock;
1016
1017 return 0;
1018
1019err_unlock:
1020 if (!resv)
1021 dma_resv_unlock(bo->base.resv);
1022
1023err_put:
1024 ttm_bo_put(bo);
1025 return ret;
1026}
1027EXPORT_SYMBOL(ttm_bo_init_reserved);
1028
1029/**
1030 * ttm_bo_init_validate
1031 *
1032 * @bdev: Pointer to a ttm_device struct.
1033 * @bo: Pointer to a ttm_buffer_object to be initialized.
1034 * @type: Requested type of buffer object.
1035 * @placement: Initial placement for buffer object.
1036 * @alignment: Data alignment in pages.
1037 * @interruptible: If needing to sleep to wait for GPU resources,
1038 * sleep interruptible.
1039 * pinned in physical memory. If this behaviour is not desired, this member
1040 * holds a pointer to a persistent shmem object. Typically, this would
1041 * point to the shmem object backing a GEM object if TTM is used to back a
1042 * GEM user interface.
1043 * @sg: Scatter-gather table.
1044 * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
1045 * @destroy: Destroy function. Use NULL for kfree().
1046 *
1047 * This function initializes a pre-allocated struct ttm_buffer_object.
1048 * As this object may be part of a larger structure, this function,
1049 * together with the @destroy function,
1050 * enables driver-specific objects derived from a ttm_buffer_object.
1051 *
1052 * On successful return, the caller owns an object kref to @bo. The kref and
1053 * list_kref are usually set to 1, but note that in some situations, other
1054 * tasks may already be holding references to @bo as well.
1055 *
1056 * If a failure occurs, the function will call the @destroy function, Thus,
1057 * after a failure, dereferencing @bo is illegal and will likely cause memory
1058 * corruption.
1059 *
1060 * Returns
1061 * -ENOMEM: Out of memory.
1062 * -EINVAL: Invalid placement flags.
1063 * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
1064 */
1065int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
1066 enum ttm_bo_type type, struct ttm_placement *placement,
1067 uint32_t alignment, bool_Bool interruptible,
1068 struct sg_table *sg, struct dma_resv *resv,
1069 void (*destroy) (struct ttm_buffer_object *))
1070{
1071 struct ttm_operation_ctx ctx = { interruptible, false0 };
1072 int ret;
1073
1074 ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
1075 sg, resv, destroy);
1076 if (ret)
1077 return ret;
1078
1079 if (!resv)
1080 ttm_bo_unreserve(bo);
1081
1082 return 0;
1083}
1084EXPORT_SYMBOL(ttm_bo_init_validate);
1085
1086/*
1087 * buffer object vm functions.
1088 */
1089
1090void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1091{
1092 struct ttm_device *bdev = bo->bdev;
1093
1094#ifdef __linux__
1095 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1096#else
1097 if (drm_mm_node_allocated(&bo->base.vma_node.vm_node)) {
16
←
Taking true branch
→
1098 struct vm_page *pg;
1099 bus_addr_t addr;
1100 paddr_t paddr;
1101 unsigned i;
1102
1103 if (bo->resource->bus.is_iomem) {
17
←
Dereference of null pointer
1104 addr = bo->resource->bus.offset;
1105 paddr = bus_space_mmap(bdev->memt, addr, 0, 0, 0)((bdev->memt)->mmap((addr), (0), (0), (0)));
1106 for (i = 0; i < bo->resource->num_pages; i++) {
1107 pg = PHYS_TO_VM_PAGE(paddr);
1108 if (pg)
1109 pmap_page_protect(pg, PROT_NONE0x00);
1110 paddr += PAGE_SIZE(1 << 12);
1111 }
1112 } else if (bo->ttm) {
1113 for (i = 0; i < bo->ttm->num_pages; i++) {
1114 pg = bo->ttm->pages[i];
1115 if (pg)
1116 pmap_page_protect(pg, PROT_NONE0x00);
1117 }
1118 }
1119 }
1120#endif
1121 ttm_mem_io_free(bdev, bo->resource);
1122}
1123EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1124
1125int ttm_bo_wait(struct ttm_buffer_object *bo,
1126 bool_Bool interruptible, bool_Bool no_wait)
1127{
1128 long timeout = 15 * HZhz;
1129
1130 if (no_wait) {
1131 if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP))
1132 return 0;
1133 else
1134 return -EBUSY16;
1135 }
1136
1137 timeout = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1138 interruptible, timeout);
1139 if (timeout < 0)
1140 return timeout;
1141
1142 if (timeout == 0)
1143 return -EBUSY16;
1144
1145 return 0;
1146}
1147EXPORT_SYMBOL(ttm_bo_wait);
1148
1149int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1150 gfp_t gfp_flags)
1151{
1152 struct ttm_place place;
1153 bool_Bool locked;
1154 int ret;
1155
1156 /*
1157 * While the bo may already reside in SYSTEM placement, set
1158 * SYSTEM as new placement to cover also the move further below.
1159 * The driver may use the fact that we're moving from SYSTEM
1160 * as an indication that we're about to swap out.
1161 */
1162 memset(&place, 0, sizeof(place))__builtin_memset((&place), (0), (sizeof(place)));
1163 place.mem_type = bo->resource->mem_type;
1164 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL((void *)0)))
1165 return -EBUSY16;
1166
1167 if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1
Assuming field 'ttm' is non-null
→
2
←
Assuming the condition is false
→
5
←
Taking false branch
→
1168 bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL(1 << 2) ||
3
←
Assuming the condition is false
→
1169 bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED(1 << 0) ||
4
←
Assuming the condition is false
→
1170 !ttm_bo_get_unless_zero(bo)) {
1171 if (locked)
1172 dma_resv_unlock(bo->base.resv);
1173 return -EBUSY16;
1174 }
1175
1176 if (bo->deleted) {
6
←
Assuming field 'deleted' is false
→
7
←
Taking false branch
→
1177 ret = ttm_bo_cleanup_refs(bo, false0, false0, locked);
1178 ttm_bo_put(bo);
1179 return ret == -EBUSY16 ? -ENOSPC28 : ret;
1180 }
1181
1182 /* TODO: Cleanup the locking */
1183 spin_unlock(&bo->bdev->lru_lock)mtx_leave(&bo->bdev->lru_lock);
1184
1185 /*
1186 * Move to system cached
1187 */
1188 if (bo->resource->mem_type != TTM_PL_SYSTEM0) {
8
←
Assuming field 'mem_type' is not equal to TTM_PL_SYSTEM
→
9
←
Taking true branch
→
1189 struct ttm_operation_ctx ctx = { false0, false0 };
1190 struct ttm_resource *evict_mem;
1191 struct ttm_place hop;
1192
1193 memset(&hop, 0, sizeof(hop))__builtin_memset((&hop), (0), (sizeof(hop)));
1194 place.mem_type = TTM_PL_SYSTEM0;
1195 ret = ttm_resource_alloc(bo, &place, &evict_mem);
10
←
Value assigned to field 'resource'
→
1196 if (unlikely(ret)__builtin_expect(!!(ret), 0))
11
←
Assuming 'ret' is 0
→
12
←
Taking false branch
→
1197 goto out;
1198
1199 ret = ttm_bo_handle_move_mem(bo, evict_mem, true1, &ctx, &hop);
13
←
Calling 'ttm_bo_handle_move_mem'
→
1200 if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) {
1201 WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n")({ int __ret = !!(ret == -82); if (__ret) printf("Unexpected multihop in swaput - likely driver bug.\n"
); __builtin_expect(!!(__ret), 0); });
1202 ttm_resource_free(bo, &evict_mem);
1203 goto out;
1204 }
1205 }
1206
1207 /*
1208 * Make sure BO is idle.
1209 */
1210 ret = ttm_bo_wait(bo, false0, false0);
1211 if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0))
1212 goto out;
1213
1214 ttm_bo_unmap_virtual(bo);
1215
1216 /*
1217 * Swap out. Buffer will be swapped in again as soon as
1218 * anyone tries to access a ttm page.
1219 */
1220 if (bo->bdev->funcs->swap_notify)
1221 bo->bdev->funcs->swap_notify(bo);
1222
1223 if (ttm_tt_is_populated(bo->ttm))
1224 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1225out:
1226
1227 /*
1228 * Unreserve without putting on LRU to avoid swapping out an
1229 * already swapped buffer.
1230 */
1231 if (locked)
1232 dma_resv_unlock(bo->base.resv);
1233 ttm_bo_put(bo);
1234 return ret == -EBUSY16 ? -ENOSPC28 : ret;
1235}
1236
1237void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1238{
1239 if (bo->ttm == NULL((void *)0))
1240 return;
1241
1242 ttm_tt_unpopulate(bo->bdev, bo->ttm);
1243 ttm_tt_destroy(bo->bdev, bo->ttm);
1244 bo->ttm = NULL((void *)0);
1245}