Bug Summary

File:dev/pci/drm/amd/amdgpu/amdgpu_fence.c
Warning:line 259, column 24
Value stored to 'adev' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name amdgpu_fence.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_fence.c
1/*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Dave Airlie
30 */
31#include <linux/seq_file.h>
32#include <linux/atomic.h>
33#include <linux/wait.h>
34#include <linux/kref.h>
35#include <linux/slab.h>
36#include <linux/firmware.h>
37#include <linux/pm_runtime.h>
38
39#include <drm/drm_debugfs.h>
40
41#include "amdgpu.h"
42#include "amdgpu_trace.h"
43
44/*
45 * Fences
46 * Fences mark an event in the GPUs pipeline and are used
47 * for GPU/CPU synchronization. When the fence is written,
48 * it is expected that all buffers associated with that fence
49 * are no longer in use by the associated ring on the GPU and
50 * that the the relevant GPU caches have been flushed.
51 */
52
53struct amdgpu_fence {
54 struct dma_fence base;
55
56 /* RB, DMA, etc. */
57 struct amdgpu_ring *ring;
58};
59
60static struct pool amdgpu_fence_slab;
61
62int amdgpu_fence_slab_init(void)
63{
64#ifdef __linux__
65 amdgpu_fence_slab = kmem_cache_create(
66 "amdgpu_fence", sizeof(struct amdgpu_fence), 0,
67 SLAB_HWCACHE_ALIGN, NULL((void *)0));
68 if (!amdgpu_fence_slab)
69 return -ENOMEM12;
70#else
71 pool_init(&amdgpu_fence_slab, sizeof(struct amdgpu_fence),
72 CACHELINESIZE64, IPL_TTY0x9, 0, "amdgpu_fence", NULL((void *)0));
73#endif
74 return 0;
75}
76
77void amdgpu_fence_slab_fini(void)
78{
79 rcu_barrier()__asm volatile("" : : : "memory");
80#ifdef __linux__
81 kmem_cache_destroy(amdgpu_fence_slab);
82#else
83 pool_destroy(&amdgpu_fence_slab);
84#endif
85}
86/*
87 * Cast helper
88 */
89static const struct dma_fence_ops amdgpu_fence_ops;
90static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
91{
92 struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base)({ const __typeof( ((struct amdgpu_fence *)0)->base ) *__mptr
= (f); (struct amdgpu_fence *)( (char *)__mptr - __builtin_offsetof
(struct amdgpu_fence, base) );});
93
94 if (__f->base.ops == &amdgpu_fence_ops)
95 return __f;
96
97 return NULL((void *)0);
98}
99
100/**
101 * amdgpu_fence_write - write a fence value
102 *
103 * @ring: ring the fence is associated with
104 * @seq: sequence number to write
105 *
106 * Writes a fence value to memory (all asics).
107 */
108static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
109{
110 struct amdgpu_fence_driver *drv = &ring->fence_drv;
111
112 if (drv->cpu_addr)
113 *drv->cpu_addr = cpu_to_le32(seq)((__uint32_t)(seq));
114}
115
116/**
117 * amdgpu_fence_read - read a fence value
118 *
119 * @ring: ring the fence is associated with
120 *
121 * Reads a fence value from memory (all asics).
122 * Returns the value of the fence read from memory.
123 */
124static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
125{
126 struct amdgpu_fence_driver *drv = &ring->fence_drv;
127 u32 seq = 0;
128
129 if (drv->cpu_addr)
130 seq = le32_to_cpu(*drv->cpu_addr)((__uint32_t)(*drv->cpu_addr));
131 else
132 seq = atomic_read(&drv->last_seq)({ typeof(*(&drv->last_seq)) __tmp = *(volatile typeof
(*(&drv->last_seq)) *)&(*(&drv->last_seq));
membar_datadep_consumer(); __tmp; });
133
134 return seq;
135}
136
137/**
138 * amdgpu_fence_emit - emit a fence on the requested ring
139 *
140 * @ring: ring the fence is associated with
141 * @f: resulting fence object
142 *
143 * Emits a fence command on the requested ring (all asics).
144 * Returns 0 on success, -ENOMEM on failure.
145 */
146int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
147 unsigned flags)
148{
149 struct amdgpu_device *adev = ring->adev;
150 struct amdgpu_fence *fence;
151 struct dma_fence __rcu **ptr;
152 uint32_t seq;
153 int r;
154
155#ifdef __linux__
156 fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL(0x0001 | 0x0004));
157#else
158 fence = pool_get(&amdgpu_fence_slab, PR_WAITOK0x0001);
159#endif
160 if (fence == NULL((void *)0))
161 return -ENOMEM12;
162
163 seq = ++ring->fence_drv.sync_seq;
164 fence->ring = ring;
165 dma_fence_init(&fence->base, &amdgpu_fence_ops,
166 &ring->fence_drv.lock,
167 adev->fence_context + ring->idx,
168 seq);
169 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,(ring)->funcs->emit_fence((ring), (ring->fence_drv.gpu_addr
), (seq), (flags | (1 << 1)))
170 seq, flags | AMDGPU_FENCE_FLAG_INT)(ring)->funcs->emit_fence((ring), (ring->fence_drv.gpu_addr
), (seq), (flags | (1 << 1)));
171 pm_runtime_get_noresume(adev_to_drm(adev)->dev);
172 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
173 if (unlikely(rcu_dereference_protected(*ptr, 1))__builtin_expect(!!((*ptr)), 0)) {
174 struct dma_fence *old;
175
176 rcu_read_lock();
177 old = dma_fence_get_rcu_safe(ptr);
178 rcu_read_unlock();
179
180 if (old) {
181 r = dma_fence_wait(old, false0);
182 dma_fence_put(old);
183 if (r)
184 return r;
185 }
186 }
187
188 /* This function can't be called concurrently anyway, otherwise
189 * emitting the fence would mess up the hardware ring buffer.
190 */
191 rcu_assign_pointer(*ptr, dma_fence_get(&fence->base))do { (*ptr) = (dma_fence_get(&fence->base)); } while(0
);
192
193 *f = &fence->base;
194
195 return 0;
196}
197
198/**
199 * amdgpu_fence_emit_polling - emit a fence on the requeste ring
200 *
201 * @ring: ring the fence is associated with
202 * @s: resulting sequence number
203 *
204 * Emits a fence command on the requested ring (all asics).
205 * Used For polling fence.
206 * Returns 0 on success, -ENOMEM on failure.
207 */
208int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
209 uint32_t timeout)
210{
211 uint32_t seq;
212 signed long r;
213
214 if (!s)
215 return -EINVAL22;
216
217 seq = ++ring->fence_drv.sync_seq;
218 r = amdgpu_fence_wait_polling(ring,
219 seq - ring->fence_drv.num_fences_mask,
220 timeout);
221 if (r < 1)
222 return -ETIMEDOUT60;
223
224 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,(ring)->funcs->emit_fence((ring), (ring->fence_drv.gpu_addr
), (seq), (0))
225 seq, 0)(ring)->funcs->emit_fence((ring), (ring->fence_drv.gpu_addr
), (seq), (0));
226
227 *s = seq;
228
229 return 0;
230}
231
232/**
233 * amdgpu_fence_schedule_fallback - schedule fallback check
234 *
235 * @ring: pointer to struct amdgpu_ring
236 *
237 * Start a timer as fallback to our interrupts.
238 */
239static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
240{
241 mod_timer(&ring->fence_drv.fallback_timer,
242 jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT(hz / 2));
243}
244
245/**
246 * amdgpu_fence_process - check for fence activity
247 *
248 * @ring: pointer to struct amdgpu_ring
249 *
250 * Checks the current fence value and calculates the last
251 * signalled fence value. Wakes the fence queue if the
252 * sequence number has increased.
253 *
254 * Returns true if fence was processed
255 */
256bool_Bool amdgpu_fence_process(struct amdgpu_ring *ring)
257{
258 struct amdgpu_fence_driver *drv = &ring->fence_drv;
259 struct amdgpu_device *adev = ring->adev;
Value stored to 'adev' during its initialization is never read
260 uint32_t seq, last_seq;
261 int r;
262
263 do {
264 last_seq = atomic_read(&ring->fence_drv.last_seq)({ typeof(*(&ring->fence_drv.last_seq)) __tmp = *(volatile
typeof(*(&ring->fence_drv.last_seq)) *)&(*(&ring
->fence_drv.last_seq)); membar_datadep_consumer(); __tmp; }
);
265 seq = amdgpu_fence_read(ring);
266
267 } while (atomic_cmpxchg(&drv->last_seq, last_seq, seq)__sync_val_compare_and_swap(&drv->last_seq, last_seq, seq
) != last_seq);
268
269 if (del_timer(&ring->fence_drv.fallback_timer)timeout_del((&ring->fence_drv.fallback_timer)) &&
270 seq != ring->fence_drv.sync_seq)
271 amdgpu_fence_schedule_fallback(ring);
272
273 if (unlikely(seq == last_seq)__builtin_expect(!!(seq == last_seq), 0))
274 return false0;
275
276 last_seq &= drv->num_fences_mask;
277 seq &= drv->num_fences_mask;
278
279 do {
280 struct dma_fence *fence, **ptr;
281
282 ++last_seq;
283 last_seq &= drv->num_fences_mask;
284 ptr = &drv->fences[last_seq];
285
286 /* There is always exactly one thread signaling this fence slot */
287 fence = rcu_dereference_protected(*ptr, 1)(*ptr);
288 RCU_INIT_POINTER(*ptr, NULL)do { (*ptr) = (((void *)0)); } while(0);
289
290 if (!fence)
291 continue;
292
293 r = dma_fence_signal(fence);
294 if (!r)
295 DMA_FENCE_TRACE(fence, "signaled from irq context\n")do {} while(0);
296 else
297 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_fence.c"
, 297); } while (0);
298
299 dma_fence_put(fence);
300 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
301 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
302 } while (last_seq != seq);
303
304 return true1;
305}
306
307/**
308 * amdgpu_fence_fallback - fallback for hardware interrupts
309 *
310 * @work: delayed work item
311 *
312 * Checks for fence activity.
313 */
314static void amdgpu_fence_fallback(void *arg)
315{
316 struct amdgpu_ring *ring = arg;
317
318 if (amdgpu_fence_process(ring))
319 DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name)printk("\0014" "[" "drm" "] " "Fence fallback timer expired on ring %s\n"
, ring->name);
320}
321
322/**
323 * amdgpu_fence_wait_empty - wait for all fences to signal
324 *
325 * @adev: amdgpu device pointer
326 * @ring: ring index the fence is associated with
327 *
328 * Wait for all fences on the requested ring to signal (all asics).
329 * Returns 0 if the fences have passed, error for all other cases.
330 */
331int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
332{
333 uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq)({ typeof(ring->fence_drv.sync_seq) __tmp = *(volatile typeof
(ring->fence_drv.sync_seq) *)&(ring->fence_drv.sync_seq
); membar_datadep_consumer(); __tmp; });
334 struct dma_fence *fence, **ptr;
335 int r;
336
337 if (!seq)
338 return 0;
339
340 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
341 rcu_read_lock();
342 fence = rcu_dereference(*ptr)(*ptr);
343 if (!fence || !dma_fence_get_rcu(fence)) {
344 rcu_read_unlock();
345 return 0;
346 }
347 rcu_read_unlock();
348
349 r = dma_fence_wait(fence, false0);
350 dma_fence_put(fence);
351 return r;
352}
353
354/**
355 * amdgpu_fence_wait_polling - busy wait for givn sequence number
356 *
357 * @ring: ring index the fence is associated with
358 * @wait_seq: sequence number to wait
359 * @timeout: the timeout for waiting in usecs
360 *
361 * Wait for all fences on the requested ring to signal (all asics).
362 * Returns left time if no timeout, 0 or minus if timeout.
363 */
364signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
365 uint32_t wait_seq,
366 signed long timeout)
367{
368 uint32_t seq;
369
370 do {
371 seq = amdgpu_fence_read(ring);
372 udelay(5);
373 timeout -= 5;
374 } while ((int32_t)(wait_seq - seq) > 0 && timeout > 0);
375
376 return timeout > 0 ? timeout : 0;
377}
378/**
379 * amdgpu_fence_count_emitted - get the count of emitted fences
380 *
381 * @ring: ring the fence is associated with
382 *
383 * Get the number of fences emitted on the requested ring (all asics).
384 * Returns the number of emitted fences on the ring. Used by the
385 * dynpm code to ring track activity.
386 */
387unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
388{
389 uint64_t emitted;
390
391 /* We are not protected by ring lock when reading the last sequence
392 * but it's ok to report slightly wrong fence count here.
393 */
394 amdgpu_fence_process(ring);
395 emitted = 0x100000000ull;
396 emitted -= atomic_read(&ring->fence_drv.last_seq)({ typeof(*(&ring->fence_drv.last_seq)) __tmp = *(volatile
typeof(*(&ring->fence_drv.last_seq)) *)&(*(&ring
->fence_drv.last_seq)); membar_datadep_consumer(); __tmp; }
);
397 emitted += READ_ONCE(ring->fence_drv.sync_seq)({ typeof(ring->fence_drv.sync_seq) __tmp = *(volatile typeof
(ring->fence_drv.sync_seq) *)&(ring->fence_drv.sync_seq
); membar_datadep_consumer(); __tmp; });
398 return lower_32_bits(emitted)((u32)(emitted));
399}
400
401/**
402 * amdgpu_fence_driver_start_ring - make the fence driver
403 * ready for use on the requested ring.
404 *
405 * @ring: ring to start the fence driver on
406 * @irq_src: interrupt source to use for this ring
407 * @irq_type: interrupt type to use for this ring
408 *
409 * Make the fence driver ready for processing (all asics).
410 * Not all asics have all rings, so each asic will only
411 * start the fence driver on the rings it has.
412 * Returns 0 for success, errors for failure.
413 */
414int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
415 struct amdgpu_irq_src *irq_src,
416 unsigned irq_type)
417{
418 struct amdgpu_device *adev = ring->adev;
419 uint64_t index;
420
421 if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {
422 ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];
423 ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);
424 } else {
425 /* put fence directly behind firmware */
426 index = roundup2(adev->uvd.fw->size, 8)(((adev->uvd.fw->size) + ((8) - 1)) & (~((__typeof(
adev->uvd.fw->size))(8) - 1)));
427 ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;
428 ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;
429 }
430 amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq)({ typeof(*(&ring->fence_drv.last_seq)) __tmp = *(volatile
typeof(*(&ring->fence_drv.last_seq)) *)&(*(&ring
->fence_drv.last_seq)); membar_datadep_consumer(); __tmp; }
));
431
432 if (irq_src)
433 amdgpu_irq_get(adev, irq_src, irq_type);
434
435 ring->fence_drv.irq_src = irq_src;
436 ring->fence_drv.irq_type = irq_type;
437 ring->fence_drv.initialized = true1;
438
439 DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",drm_dev_dbg(adev->dev, DRM_UT_CORE, "fence driver on ring %s use gpu addr 0x%016llx\n"
, ring->name, ring->fence_drv.gpu_addr)
440 ring->name, ring->fence_drv.gpu_addr)drm_dev_dbg(adev->dev, DRM_UT_CORE, "fence driver on ring %s use gpu addr 0x%016llx\n"
, ring->name, ring->fence_drv.gpu_addr);
441 return 0;
442}
443
444/**
445 * amdgpu_fence_driver_init_ring - init the fence driver
446 * for the requested ring.
447 *
448 * @ring: ring to init the fence driver on
449 * @num_hw_submission: number of entries on the hardware queue
450 *
451 * Init the fence driver for the requested ring (all asics).
452 * Helper function for amdgpu_fence_driver_init().
453 */
454int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
455 unsigned num_hw_submission)
456{
457 struct amdgpu_device *adev = ring->adev;
458 long timeout;
459 int r;
460
461 if (!adev)
462 return -EINVAL22;
463
464 if (!is_power_of_2(num_hw_submission)(((num_hw_submission) != 0) && (((num_hw_submission) -
1) & (num_hw_submission)) == 0))
465 return -EINVAL22;
466
467 ring->fence_drv.cpu_addr = NULL((void *)0);
468 ring->fence_drv.gpu_addr = 0;
469 ring->fence_drv.sync_seq = 0;
470 atomic_set(&ring->fence_drv.last_seq, 0)({ typeof(*(&ring->fence_drv.last_seq)) __tmp = ((0));
*(volatile typeof(*(&ring->fence_drv.last_seq)) *)&
(*(&ring->fence_drv.last_seq)) = __tmp; __tmp; });
471 ring->fence_drv.initialized = false0;
472
473#ifdef __linux__
474 timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
475#else
476 timeout_set(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback,
477 ring);
478#endif
479
480 ring->fence_drv.num_fences_mask = num_hw_submission * 2 - 1;
481 mtx_init(&ring->fence_drv.lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&ring->
fence_drv.lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
482 ring->fence_drv.fences = kcalloc(num_hw_submission * 2, sizeof(void *),
483 GFP_KERNEL(0x0001 | 0x0004));
484 if (!ring->fence_drv.fences)
485 return -ENOMEM12;
486
487 /* No need to setup the GPU scheduler for rings that don't need it */
488 if (!ring->no_scheduler) {
489 switch (ring->funcs->type) {
490 case AMDGPU_RING_TYPE_GFX:
491 timeout = adev->gfx_timeout;
492 break;
493 case AMDGPU_RING_TYPE_COMPUTE:
494 timeout = adev->compute_timeout;
495 break;
496 case AMDGPU_RING_TYPE_SDMA:
497 timeout = adev->sdma_timeout;
498 break;
499 default:
500 timeout = adev->video_timeout;
501 break;
502 }
503
504 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
505 num_hw_submission, amdgpu_job_hang_limit,
506 timeout, ring->name);
507 if (r) {
508 DRM_ERROR("Failed to create scheduler on ring %s.\n",__drm_err("Failed to create scheduler on ring %s.\n", ring->
name)
509 ring->name)__drm_err("Failed to create scheduler on ring %s.\n", ring->
name);
510 return r;
511 }
512 }
513
514 return 0;
515}
516
517/**
518 * amdgpu_fence_driver_init - init the fence driver
519 * for all possible rings.
520 *
521 * @adev: amdgpu device pointer
522 *
523 * Init the fence driver for all possible rings (all asics).
524 * Not all asics have all rings, so each asic will only
525 * start the fence driver on the rings it has using
526 * amdgpu_fence_driver_start_ring().
527 * Returns 0 for success.
528 */
529int amdgpu_fence_driver_init(struct amdgpu_device *adev)
530{
531 return 0;
532}
533
534/**
535 * amdgpu_fence_driver_fini - tear down the fence driver
536 * for all possible rings.
537 *
538 * @adev: amdgpu device pointer
539 *
540 * Tear down the fence driver for all possible rings (all asics).
541 */
542void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
543{
544 unsigned i, j;
545 int r;
546
547 for (i = 0; i < AMDGPU_MAX_RINGS28; i++) {
548 struct amdgpu_ring *ring = adev->rings[i];
549
550 if (!ring || !ring->fence_drv.initialized)
551 continue;
552 if (!ring->no_scheduler)
553 drm_sched_fini(&ring->sched);
554 r = amdgpu_fence_wait_empty(ring);
555 if (r) {
556 /* no need to trigger GPU reset as we are unloading */
557 amdgpu_fence_driver_force_completion(ring);
558 }
559 if (ring->fence_drv.irq_src)
560 amdgpu_irq_put(adev, ring->fence_drv.irq_src,
561 ring->fence_drv.irq_type);
562
563 del_timer_sync(&ring->fence_drv.fallback_timer)timeout_del_barrier((&ring->fence_drv.fallback_timer));
564 for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
565 dma_fence_put(ring->fence_drv.fences[j]);
566 kfree(ring->fence_drv.fences);
567 ring->fence_drv.fences = NULL((void *)0);
568 ring->fence_drv.initialized = false0;
569 }
570}
571
572/**
573 * amdgpu_fence_driver_suspend - suspend the fence driver
574 * for all possible rings.
575 *
576 * @adev: amdgpu device pointer
577 *
578 * Suspend the fence driver for all possible rings (all asics).
579 */
580void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
581{
582 int i, r;
583
584 for (i = 0; i < AMDGPU_MAX_RINGS28; i++) {
585 struct amdgpu_ring *ring = adev->rings[i];
586 if (!ring || !ring->fence_drv.initialized)
587 continue;
588
589 /* wait for gpu to finish processing current batch */
590 r = amdgpu_fence_wait_empty(ring);
591 if (r) {
592 /* delay GPU reset to resume */
593 amdgpu_fence_driver_force_completion(ring);
594 }
595
596 /* disable the interrupt */
597 if (ring->fence_drv.irq_src)
598 amdgpu_irq_put(adev, ring->fence_drv.irq_src,
599 ring->fence_drv.irq_type);
600 }
601}
602
603/**
604 * amdgpu_fence_driver_resume - resume the fence driver
605 * for all possible rings.
606 *
607 * @adev: amdgpu device pointer
608 *
609 * Resume the fence driver for all possible rings (all asics).
610 * Not all asics have all rings, so each asic will only
611 * start the fence driver on the rings it has using
612 * amdgpu_fence_driver_start_ring().
613 * Returns 0 for success.
614 */
615void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
616{
617 int i;
618
619 for (i = 0; i < AMDGPU_MAX_RINGS28; i++) {
620 struct amdgpu_ring *ring = adev->rings[i];
621 if (!ring || !ring->fence_drv.initialized)
622 continue;
623
624 /* enable the interrupt */
625 if (ring->fence_drv.irq_src)
626 amdgpu_irq_get(adev, ring->fence_drv.irq_src,
627 ring->fence_drv.irq_type);
628 }
629}
630
631/**
632 * amdgpu_fence_driver_force_completion - force signal latest fence of ring
633 *
634 * @ring: fence of the ring to signal
635 *
636 */
637void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
638{
639 amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
640 amdgpu_fence_process(ring);
641}
642
643/*
644 * Common fence implementation
645 */
646
647static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
648{
649 return "amdgpu";
650}
651
652static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
653{
654 struct amdgpu_fence *fence = to_amdgpu_fence(f);
655 return (const char *)fence->ring->name;
656}
657
658/**
659 * amdgpu_fence_enable_signaling - enable signalling on fence
660 * @fence: fence
661 *
662 * This function is called with fence_queue lock held, and adds a callback
663 * to fence_queue that checks if this fence is signaled, and if so it
664 * signals the fence and removes itself.
665 */
666static bool_Bool amdgpu_fence_enable_signaling(struct dma_fence *f)
667{
668 struct amdgpu_fence *fence = to_amdgpu_fence(f);
669 struct amdgpu_ring *ring = fence->ring;
670
671 if (!timer_pending(&ring->fence_drv.fallback_timer)(((&ring->fence_drv.fallback_timer))->to_flags &
0x02))
672 amdgpu_fence_schedule_fallback(ring);
673
674 DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx)do {} while(0);
675
676 return true1;
677}
678
679/**
680 * amdgpu_fence_free - free up the fence memory
681 *
682 * @rcu: RCU callback head
683 *
684 * Free up the fence memory after the RCU grace period.
685 */
686static void amdgpu_fence_free(struct rcu_head *rcu)
687{
688 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu)({ const __typeof( ((struct dma_fence *)0)->rcu ) *__mptr =
(rcu); (struct dma_fence *)( (char *)__mptr - __builtin_offsetof
(struct dma_fence, rcu) );});
689 struct amdgpu_fence *fence = to_amdgpu_fence(f);
690#ifdef __linux__
691 kmem_cache_free(amdgpu_fence_slab, fence);
692#else
693 pool_put(&amdgpu_fence_slab, fence);
694#endif
695}
696
697/**
698 * amdgpu_fence_release - callback that fence can be freed
699 *
700 * @fence: fence
701 *
702 * This function is called when the reference count becomes zero.
703 * It just RCU schedules freeing up the fence.
704 */
705static void amdgpu_fence_release(struct dma_fence *f)
706{
707 call_rcu(&f->rcu, amdgpu_fence_free);
708}
709
710static const struct dma_fence_ops amdgpu_fence_ops = {
711 .get_driver_name = amdgpu_fence_get_driver_name,
712 .get_timeline_name = amdgpu_fence_get_timeline_name,
713 .enable_signaling = amdgpu_fence_enable_signaling,
714 .release = amdgpu_fence_release,
715};
716
717/*
718 * Fence debugfs
719 */
720#if defined(CONFIG_DEBUG_FS)
721static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
722{
723 struct drm_info_node *node = (struct drm_info_node *)m->private;
724 struct drm_device *dev = node->minor->dev;
725 struct amdgpu_device *adev = drm_to_adev(dev);
726 int i;
727
728 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
729 struct amdgpu_ring *ring = adev->rings[i];
730 if (!ring || !ring->fence_drv.initialized)
731 continue;
732
733 amdgpu_fence_process(ring);
734
735 seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
736 seq_printf(m, "Last signaled fence 0x%08x\n",
737 atomic_read(&ring->fence_drv.last_seq)({ typeof(*(&ring->fence_drv.last_seq)) __tmp = *(volatile
typeof(*(&ring->fence_drv.last_seq)) *)&(*(&ring
->fence_drv.last_seq)); membar_datadep_consumer(); __tmp; }
));
738 seq_printf(m, "Last emitted 0x%08x\n",
739 ring->fence_drv.sync_seq);
740
741 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||
742 ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {
743 seq_printf(m, "Last signaled trailing fence 0x%08x\n",
744 le32_to_cpu(*ring->trail_fence_cpu_addr)((__uint32_t)(*ring->trail_fence_cpu_addr)));
745 seq_printf(m, "Last emitted 0x%08x\n",
746 ring->trail_seq);
747 }
748
749 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
750 continue;
751
752 /* set in CP_VMID_PREEMPT and preemption occurred */
753 seq_printf(m, "Last preempted 0x%08x\n",
754 le32_to_cpu(*(ring->fence_drv.cpu_addr + 2))((__uint32_t)(*(ring->fence_drv.cpu_addr + 2))));
755 /* set in CP_VMID_RESET and reset occurred */
756 seq_printf(m, "Last reset 0x%08x\n",
757 le32_to_cpu(*(ring->fence_drv.cpu_addr + 4))((__uint32_t)(*(ring->fence_drv.cpu_addr + 4))));
758 /* Both preemption and reset occurred */
759 seq_printf(m, "Last both 0x%08x\n",
760 le32_to_cpu(*(ring->fence_drv.cpu_addr + 6))((__uint32_t)(*(ring->fence_drv.cpu_addr + 6))));
761 }
762 return 0;
763}
764
765/**
766 * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
767 *
768 * Manually trigger a gpu reset at the next fence wait.
769 */
770static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data)
771{
772 struct drm_info_node *node = (struct drm_info_node *) m->private;
773 struct drm_device *dev = node->minor->dev;
774 struct amdgpu_device *adev = drm_to_adev(dev);
775 int r;
776
777 r = pm_runtime_get_sync(dev->dev);
778 if (r < 0) {
779 pm_runtime_put_autosuspend(dev->dev);
780 return 0;
781 }
782
783 seq_printf(m, "gpu recover\n");
784 amdgpu_device_gpu_recover(adev, NULL((void *)0));
785
786 pm_runtime_mark_last_busy(dev->dev);
787 pm_runtime_put_autosuspend(dev->dev);
788
789 return 0;
790}
791
792static const struct drm_info_list amdgpu_debugfs_fence_list[] = {
793 {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL((void *)0)},
794 {"amdgpu_gpu_recover", &amdgpu_debugfs_gpu_recover, 0, NULL((void *)0)}
795};
796
797static const struct drm_info_list amdgpu_debugfs_fence_list_sriov[] = {
798 {"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL((void *)0)},
799};
800#endif
801
802int amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
803{
804#if defined(CONFIG_DEBUG_FS)
805 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
806 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list_sriov,
807 ARRAY_SIZE(amdgpu_debugfs_fence_list_sriov)(sizeof((amdgpu_debugfs_fence_list_sriov)) / sizeof((amdgpu_debugfs_fence_list_sriov
)[0])));
808 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list,
809 ARRAY_SIZE(amdgpu_debugfs_fence_list)(sizeof((amdgpu_debugfs_fence_list)) / sizeof((amdgpu_debugfs_fence_list
)[0])));
810#else
811 return 0;
812#endif
813}
814