Bug Summary

File:dev/pci/drm/i915/i915_scheduler.c
Warning:line 105, column 4
Value stored to 'first' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name i915_scheduler.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/i915/i915_scheduler.c
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7#include <linux/mutex.h>
8
9#include "i915_drv.h"
10#include "i915_globals.h"
11#include "i915_request.h"
12#include "i915_scheduler.h"
13
14static struct i915_global_scheduler {
15 struct i915_global base;
16#ifdef __linux__
17 struct kmem_cache *slab_dependencies;
18 struct kmem_cache *slab_priorities;
19#else
20 struct pool slab_dependencies;
21 struct pool slab_priorities;
22#endif
23} global;
24
25static DEFINE_SPINLOCK(schedule_lock)struct mutex schedule_lock = { ((void *)0), ((((0x9)) > 0x0
&& ((0x9)) < 0x9) ? 0x9 : ((0x9))), 0x0 }
;
26
27static const struct i915_request *
28node_to_request(const struct i915_sched_node *node)
29{
30 return container_of(node, const struct i915_request, sched)({ const __typeof( ((const struct i915_request *)0)->sched
) *__mptr = (node); (const struct i915_request *)( (char *)__mptr
- __builtin_offsetof(const struct i915_request, sched) );})
;
31}
32
33static inline bool_Bool node_started(const struct i915_sched_node *node)
34{
35 return i915_request_started(node_to_request(node));
36}
37
38static inline bool_Bool node_signaled(const struct i915_sched_node *node)
39{
40 return i915_request_completed(node_to_request(node));
41}
42
43static inline struct i915_priolist *to_priolist(struct rb_node *rb)
44{
45 return rb_entry(rb, struct i915_priolist, node)({ const __typeof( ((struct i915_priolist *)0)->node ) *__mptr
= (rb); (struct i915_priolist *)( (char *)__mptr - __builtin_offsetof
(struct i915_priolist, node) );})
;
46}
47
48static void assert_priolists(struct intel_engine_execlists * const execlists)
49{
50 struct rb_node *rb;
51 long last_prio, i;
52
53 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)0)
54 return;
55
56 GEM_BUG_ON(rb_first_cached(&execlists->queue) !=((void)0)
57 rb_first(&execlists->queue.rb_root))((void)0);
58
59 last_prio = INT_MAX0x7fffffff;
60 for (rb = rb_first_cached(&execlists->queue)linux_root_RB_MINMAX((struct linux_root *)(&(&execlists
->queue)->rb_root), -1)
; rb; rb = rb_next(rb)linux_root_RB_NEXT((rb))) {
61 const struct i915_priolist *p = to_priolist(rb);
62
63 GEM_BUG_ON(p->priority > last_prio)((void)0);
64 last_prio = p->priority;
65
66 GEM_BUG_ON(!p->used)((void)0);
67 for (i = 0; i < ARRAY_SIZE(p->requests)(sizeof((p->requests)) / sizeof((p->requests)[0])); i++) {
68 if (list_empty(&p->requests[i]))
69 continue;
70
71 GEM_BUG_ON(!(p->used & BIT(i)))((void)0);
72 }
73 }
74}
75
76struct list_head *
77i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
78{
79 struct intel_engine_execlists * const execlists = &engine->execlists;
80 struct i915_priolist *p;
81 struct rb_node **parent, *rb;
82 bool_Bool first = true1;
83 int idx, i;
84
85 lockdep_assert_held(&engine->active.lock)do { (void)(&engine->active.lock); } while(0);
86 assert_priolists(execlists);
87
88 /* buckets sorted from highest [in slot 0] to lowest priority */
89 idx = I915_PRIORITY_COUNT(1UL << (0)) - (prio & I915_PRIORITY_MASK((1UL << (0)) - 1)) - 1;
90 prio >>= I915_USER_PRIORITY_SHIFT0;
91 if (unlikely(execlists->no_priolist)__builtin_expect(!!(execlists->no_priolist), 0))
92 prio = I915_PRIORITY_NORMAL;
93
94find_priolist:
95 /* most positive priority is scheduled first, equal priorities fifo */
96 rb = NULL((void *)0);
97 parent = &execlists->queue.rb_root.rb_node;
98 while (*parent) {
99 rb = *parent;
100 p = to_priolist(rb);
101 if (prio > p->priority) {
102 parent = &rb->rb_left__entry.rbe_left;
103 } else if (prio < p->priority) {
104 parent = &rb->rb_right__entry.rbe_right;
105 first = false0;
Value stored to 'first' is never read
106 } else {
107 goto out;
108 }
109 }
110
111 if (prio == I915_PRIORITY_NORMAL) {
112 p = &execlists->default_priolist;
113 } else {
114#ifdef __linux__
115 p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC0x0002);
116#else
117 p = pool_get(&global.slab_priorities, PR_NOWAIT0x0002);
118#endif
119 /* Convert an allocation failure to a priority bump */
120 if (unlikely(!p)__builtin_expect(!!(!p), 0)) {
121 prio = I915_PRIORITY_NORMAL; /* recurses just once */
122
123 /* To maintain ordering with all rendering, after an
124 * allocation failure we have to disable all scheduling.
125 * Requests will then be executed in fifo, and schedule
126 * will ensure that dependencies are emitted in fifo.
127 * There will be still some reordering with existing
128 * requests, so if userspace lied about their
129 * dependencies that reordering may be visible.
130 */
131 execlists->no_priolist = true1;
132 goto find_priolist;
133 }
134 }
135
136 p->priority = prio;
137 for (i = 0; i < ARRAY_SIZE(p->requests)(sizeof((p->requests)) / sizeof((p->requests)[0])); i++)
138 INIT_LIST_HEAD(&p->requests[i]);
139 rb_link_node(&p->node, rb, parent);
140 rb_insert_color_cached(&p->node, &execlists->queue, first)linux_root_RB_INSERT_COLOR((struct linux_root *)(&(&execlists
->queue)->rb_root), (&p->node))
;
141 p->used = 0;
142
143out:
144 p->used |= BIT(idx)(1UL << (idx));
145 return &p->requests[idx];
146}
147
148void __i915_priolist_free(struct i915_priolist *p)
149{
150#ifdef __linux__
151 kmem_cache_free(global.slab_priorities, p);
152#else
153 pool_put(&global.slab_priorities, p);
154#endif
155}
156
157struct sched_cache {
158 struct list_head *priolist;
159};
160
161static struct intel_engine_cs *
162sched_lock_engine(const struct i915_sched_node *node,
163 struct intel_engine_cs *locked,
164 struct sched_cache *cache)
165{
166 const struct i915_request *rq = node_to_request(node);
167 struct intel_engine_cs *engine;
168
169 GEM_BUG_ON(!locked)((void)0);
170
171 /*
172 * Virtual engines complicate acquiring the engine timeline lock,
173 * as their rq->engine pointer is not stable until under that
174 * engine lock. The simple ploy we use is to take the lock then
175 * check that the rq still belongs to the newly locked engine.
176 */
177 while (locked != (engine = READ_ONCE(rq->engine)({ typeof(rq->engine) __tmp = *(volatile typeof(rq->engine
) *)&(rq->engine); membar_datadep_consumer(); __tmp; }
)
)) {
178 spin_unlock(&locked->active.lock)mtx_leave(&locked->active.lock);
179 memset(cache, 0, sizeof(*cache))__builtin_memset((cache), (0), (sizeof(*cache)));
180 spin_lock(&engine->active.lock)mtx_enter(&engine->active.lock);
181 locked = engine;
182 }
183
184 GEM_BUG_ON(locked != engine)((void)0);
185 return locked;
186}
187
188static inline int rq_prio(const struct i915_request *rq)
189{
190 return rq->sched.attr.priority;
191}
192
193static inline bool_Bool need_preempt(int prio, int active)
194{
195 /*
196 * Allow preemption of low -> normal -> high, but we do
197 * not allow low priority tasks to preempt other low priority
198 * tasks under the impression that latency for low priority
199 * tasks does not matter (as much as background throughput),
200 * so kiss.
201 */
202 return prio >= max(I915_PRIORITY_NORMAL, active)(((I915_PRIORITY_NORMAL)>(active))?(I915_PRIORITY_NORMAL):
(active))
;
203}
204
205static void kick_submission(struct intel_engine_cs *engine,
206 const struct i915_request *rq,
207 int prio)
208{
209 const struct i915_request *inflight;
210
211 /*
212 * We only need to kick the tasklet once for the high priority
213 * new context we add into the queue.
214 */
215 if (prio <= engine->execlists.queue_priority_hint)
216 return;
217
218 rcu_read_lock();
219
220 /* Nothing currently active? We're overdue for a submission! */
221 inflight = execlists_active(&engine->execlists);
222 if (!inflight)
223 goto unlock;
224
225 /*
226 * If we are already the currently executing context, don't
227 * bother evaluating if we should preempt ourselves.
228 */
229 if (inflight->context == rq->context)
230 goto unlock;
231
232 ENGINE_TRACE(engine,do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0)
233 "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0)
234 prio,do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0)
235 rq->fence.context, rq->fence.seqno,do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0)
236 inflight->fence.context, inflight->fence.seqno,do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0)
237 inflight->sched.attr.priority)do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (engine); do { } while (0); } while (0)
;
238
239 engine->execlists.queue_priority_hint = prio;
240 if (need_preempt(prio, rq_prio(inflight)))
241 tasklet_hi_schedule(&engine->execlists.tasklet);
242
243unlock:
244 rcu_read_unlock();
245}
246
247static void __i915_schedule(struct i915_sched_node *node,
248 const struct i915_sched_attr *attr)
249{
250 const int prio = max(attr->priority, node->attr.priority)(((attr->priority)>(node->attr.priority))?(attr->
priority):(node->attr.priority))
;
251 struct intel_engine_cs *engine;
252 struct i915_dependency *dep, *p;
253 struct i915_dependency stack;
254 struct sched_cache cache;
255 DRM_LIST_HEAD(dfs)struct list_head dfs = { &(dfs), &(dfs) };
256
257 /* Needed in order to use the temporary link inside i915_dependency */
258 lockdep_assert_held(&schedule_lock)do { (void)(&schedule_lock); } while(0);
259 GEM_BUG_ON(prio == I915_PRIORITY_INVALID)((void)0);
260
261 if (node_signaled(node))
262 return;
263
264 stack.signaler = node;
265 list_add(&stack.dfs_link, &dfs);
266
267 /*
268 * Recursively bump all dependent priorities to match the new request.
269 *
270 * A naive approach would be to use recursion:
271 * static void update_priorities(struct i915_sched_node *node, prio) {
272 * list_for_each_entry(dep, &node->signalers_list, signal_link)
273 * update_priorities(dep->signal, prio)
274 * queue_request(node);
275 * }
276 * but that may have unlimited recursion depth and so runs a very
277 * real risk of overunning the kernel stack. Instead, we build
278 * a flat list of all dependencies starting with the current request.
279 * As we walk the list of dependencies, we add all of its dependencies
280 * to the end of the list (this may include an already visited
281 * request) and continue to walk onwards onto the new dependencies. The
282 * end result is a topological list of requests in reverse order, the
283 * last element in the list is the request we must execute first.
284 */
285 list_for_each_entry(dep, &dfs, dfs_link)for (dep = ({ const __typeof( ((__typeof(*dep) *)0)->dfs_link
) *__mptr = ((&dfs)->next); (__typeof(*dep) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*dep), dfs_link) );})
; &dep->dfs_link != (&dfs); dep = ({ const __typeof
( ((__typeof(*dep) *)0)->dfs_link ) *__mptr = (dep->dfs_link
.next); (__typeof(*dep) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*dep), dfs_link) );}))
{
286 struct i915_sched_node *node = dep->signaler;
287
288 /* If we are already flying, we know we have no signalers */
289 if (node_started(node))
290 continue;
291
292 /*
293 * Within an engine, there can be no cycle, but we may
294 * refer to the same dependency chain multiple times
295 * (redundant dependencies are not eliminated) and across
296 * engines.
297 */
298 list_for_each_entry(p, &node->signalers_list, signal_link)for (p = ({ const __typeof( ((__typeof(*p) *)0)->signal_link
) *__mptr = ((&node->signalers_list)->next); (__typeof
(*p) *)( (char *)__mptr - __builtin_offsetof(__typeof(*p), signal_link
) );}); &p->signal_link != (&node->signalers_list
); p = ({ const __typeof( ((__typeof(*p) *)0)->signal_link
) *__mptr = (p->signal_link.next); (__typeof(*p) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*p), signal_link) );}
))
{
299 GEM_BUG_ON(p == dep)((void)0); /* no cycles! */
300
301 if (node_signaled(p->signaler))
302 continue;
303
304 if (prio > READ_ONCE(p->signaler->attr.priority)({ typeof(p->signaler->attr.priority) __tmp = *(volatile
typeof(p->signaler->attr.priority) *)&(p->signaler
->attr.priority); membar_datadep_consumer(); __tmp; })
)
305 list_move_tail(&p->dfs_link, &dfs);
306 }
307 }
308
309 /*
310 * If we didn't need to bump any existing priorities, and we haven't
311 * yet submitted this request (i.e. there is no potential race with
312 * execlists_submit_request()), we can set our own priority and skip
313 * acquiring the engine locks.
314 */
315 if (node->attr.priority == I915_PRIORITY_INVALID((-0x7fffffff-1) | (u8)((1UL << (0)) - 1))) {
316 GEM_BUG_ON(!list_empty(&node->link))((void)0);
317 node->attr = *attr;
318
319 if (stack.dfs_link.next == stack.dfs_link.prev)
320 return;
321
322 __list_del_entry(&stack.dfs_link)list_del(&stack.dfs_link);
323 }
324
325 memset(&cache, 0, sizeof(cache))__builtin_memset((&cache), (0), (sizeof(cache)));
326 engine = node_to_request(node)->engine;
327 spin_lock(&engine->active.lock)mtx_enter(&engine->active.lock);
328
329 /* Fifo and depth-first replacement ensure our deps execute before us */
330 engine = sched_lock_engine(node, engine, &cache);
331 list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link)for (dep = ({ const __typeof( ((__typeof(*dep) *)0)->dfs_link
) *__mptr = ((&dfs)->prev); (__typeof(*dep) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*dep), dfs_link) );})
, p = ({ const __typeof( ((__typeof(*dep) *)0)->dfs_link )
*__mptr = ((dep)->dfs_link.prev); (__typeof(*dep) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*dep), dfs_link) );})
; &(dep)->dfs_link != (&dfs); dep = p, p = ({ const
__typeof( ((__typeof(*p) *)0)->dfs_link ) *__mptr = (p->
dfs_link.prev); (__typeof(*p) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*p), dfs_link) );}))
{
332 INIT_LIST_HEAD(&dep->dfs_link);
333
334 node = dep->signaler;
335 engine = sched_lock_engine(node, engine, &cache);
336 lockdep_assert_held(&engine->active.lock)do { (void)(&engine->active.lock); } while(0);
337
338 /* Recheck after acquiring the engine->timeline.lock */
339 if (prio <= node->attr.priority || node_signaled(node))
340 continue;
341
342 GEM_BUG_ON(node_to_request(node)->engine != engine)((void)0);
343
344 WRITE_ONCE(node->attr.priority, prio)({ typeof(node->attr.priority) __tmp = (prio); *(volatile typeof
(node->attr.priority) *)&(node->attr.priority) = __tmp
; __tmp; })
;
345
346 /*
347 * Once the request is ready, it will be placed into the
348 * priority lists and then onto the HW runlist. Before the
349 * request is ready, it does not contribute to our preemption
350 * decisions and we can safely ignore it, as it will, and
351 * any preemption required, be dealt with upon submission.
352 * See engine->submit_request()
353 */
354 if (list_empty(&node->link))
355 continue;
356
357 if (i915_request_in_priority_queue(node_to_request(node))) {
358 if (!cache.priolist)
359 cache.priolist =
360 i915_sched_lookup_priolist(engine,
361 prio);
362 list_move_tail(&node->link, cache.priolist);
363 }
364
365 /* Defer (tasklet) submission until after all of our updates. */
366 kick_submission(engine, node_to_request(node), prio);
367 }
368
369 spin_unlock(&engine->active.lock)mtx_leave(&engine->active.lock);
370}
371
372void i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr)
373{
374 spin_lock_irq(&schedule_lock)mtx_enter(&schedule_lock);
375 __i915_schedule(&rq->sched, attr);
376 spin_unlock_irq(&schedule_lock)mtx_leave(&schedule_lock);
377}
378
379static void __bump_priority(struct i915_sched_node *node, unsigned int bump)
380{
381 struct i915_sched_attr attr = node->attr;
382
383 if (attr.priority & bump)
384 return;
385
386 attr.priority |= bump;
387 __i915_schedule(node, &attr);
388}
389
390void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump)
391{
392 unsigned long flags;
393
394 GEM_BUG_ON(bump & ~I915_PRIORITY_MASK)((void)0);
395 if (READ_ONCE(rq->sched.attr.priority)({ typeof(rq->sched.attr.priority) __tmp = *(volatile typeof
(rq->sched.attr.priority) *)&(rq->sched.attr.priority
); membar_datadep_consumer(); __tmp; })
& bump)
396 return;
397
398 spin_lock_irqsave(&schedule_lock, flags)do { flags = 0; mtx_enter(&schedule_lock); } while (0);
399 __bump_priority(&rq->sched, bump);
400 spin_unlock_irqrestore(&schedule_lock, flags)do { (void)(flags); mtx_leave(&schedule_lock); } while (0
)
;
401}
402
403void i915_sched_node_init(struct i915_sched_node *node)
404{
405 INIT_LIST_HEAD(&node->signalers_list);
406 INIT_LIST_HEAD(&node->waiters_list);
407 INIT_LIST_HEAD(&node->link);
408
409 i915_sched_node_reinit(node);
410}
411
412void i915_sched_node_reinit(struct i915_sched_node *node)
413{
414 node->attr.priority = I915_PRIORITY_INVALID((-0x7fffffff-1) | (u8)((1UL << (0)) - 1));
415 node->semaphores = 0;
416 node->flags = 0;
417
418 GEM_BUG_ON(!list_empty(&node->signalers_list))((void)0);
419 GEM_BUG_ON(!list_empty(&node->waiters_list))((void)0);
420 GEM_BUG_ON(!list_empty(&node->link))((void)0);
421}
422
423static struct i915_dependency *
424i915_dependency_alloc(void)
425{
426#ifdef __linux__
427 return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL(0x0001 | 0x0004));
428#else
429 return pool_get(&global.slab_dependencies, PR_WAITOK0x0001);
430#endif
431}
432
433static void
434i915_dependency_free(struct i915_dependency *dep)
435{
436#ifdef __linux__
437 kmem_cache_free(global.slab_dependencies, dep);
438#else
439 pool_put(&global.slab_dependencies, dep);
440#endif
441}
442
443bool_Bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
444 struct i915_sched_node *signal,
445 struct i915_dependency *dep,
446 unsigned long flags)
447{
448 bool_Bool ret = false0;
449
450 spin_lock_irq(&schedule_lock)mtx_enter(&schedule_lock);
451
452 if (!node_signaled(signal)) {
453 INIT_LIST_HEAD(&dep->dfs_link);
454 dep->signaler = signal;
455 dep->waiter = node;
456 dep->flags = flags;
457
458 /* All set, now publish. Beware the lockless walkers. */
459 list_add_rcu(&dep->signal_link, &node->signalers_list)list_add(&dep->signal_link, &node->signalers_list
)
;
460 list_add_rcu(&dep->wait_link, &signal->waiters_list)list_add(&dep->wait_link, &signal->waiters_list
)
;
461
462 /* Propagate the chains */
463 node->flags |= signal->flags;
464 ret = true1;
465 }
466
467 spin_unlock_irq(&schedule_lock)mtx_leave(&schedule_lock);
468
469 return ret;
470}
471
472int i915_sched_node_add_dependency(struct i915_sched_node *node,
473 struct i915_sched_node *signal,
474 unsigned long flags)
475{
476 struct i915_dependency *dep;
477
478 dep = i915_dependency_alloc();
479 if (!dep)
480 return -ENOMEM12;
481
482 local_bh_disable();
483
484 if (!__i915_sched_node_add_dependency(node, signal, dep,
485 flags | I915_DEPENDENCY_ALLOC(1UL << (0))))
486 i915_dependency_free(dep);
487
488 local_bh_enable(); /* kick submission tasklet */
489
490 return 0;
491}
492
493void i915_sched_node_fini(struct i915_sched_node *node)
494{
495 struct i915_dependency *dep, *tmp;
496
497 spin_lock_irq(&schedule_lock)mtx_enter(&schedule_lock);
498
499 /*
500 * Everyone we depended upon (the fences we wait to be signaled)
501 * should retire before us and remove themselves from our list.
502 * However, retirement is run independently on each timeline and
503 * so we may be called out-of-order.
504 */
505 list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link)for (dep = ({ const __typeof( ((__typeof(*dep) *)0)->signal_link
) *__mptr = ((&node->signalers_list)->next); (__typeof
(*dep) *)( (char *)__mptr - __builtin_offsetof(__typeof(*dep)
, signal_link) );}), tmp = ({ const __typeof( ((__typeof(*dep
) *)0)->signal_link ) *__mptr = (dep->signal_link.next)
; (__typeof(*dep) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*dep), signal_link) );}); &dep->signal_link != (&
node->signalers_list); dep = tmp, tmp = ({ const __typeof(
((__typeof(*tmp) *)0)->signal_link ) *__mptr = (tmp->signal_link
.next); (__typeof(*tmp) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp), signal_link) );}))
{
506 GEM_BUG_ON(!list_empty(&dep->dfs_link))((void)0);
507
508 list_del_rcu(&dep->wait_link)list_del(&dep->wait_link);
509 if (dep->flags & I915_DEPENDENCY_ALLOC(1UL << (0)))
510 i915_dependency_free(dep);
511 }
512 INIT_LIST_HEAD(&node->signalers_list);
513
514 /* Remove ourselves from everyone who depends upon us */
515 list_for_each_entry_safe(dep, tmp, &node->waiters_list, wait_link)for (dep = ({ const __typeof( ((__typeof(*dep) *)0)->wait_link
) *__mptr = ((&node->waiters_list)->next); (__typeof
(*dep) *)( (char *)__mptr - __builtin_offsetof(__typeof(*dep)
, wait_link) );}), tmp = ({ const __typeof( ((__typeof(*dep) *
)0)->wait_link ) *__mptr = (dep->wait_link.next); (__typeof
(*dep) *)( (char *)__mptr - __builtin_offsetof(__typeof(*dep)
, wait_link) );}); &dep->wait_link != (&node->waiters_list
); dep = tmp, tmp = ({ const __typeof( ((__typeof(*tmp) *)0)->
wait_link ) *__mptr = (tmp->wait_link.next); (__typeof(*tmp
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*tmp), wait_link
) );}))
{
516 GEM_BUG_ON(dep->signaler != node)((void)0);
517 GEM_BUG_ON(!list_empty(&dep->dfs_link))((void)0);
518
519 list_del_rcu(&dep->signal_link)list_del(&dep->signal_link);
520 if (dep->flags & I915_DEPENDENCY_ALLOC(1UL << (0)))
521 i915_dependency_free(dep);
522 }
523 INIT_LIST_HEAD(&node->waiters_list);
524
525 spin_unlock_irq(&schedule_lock)mtx_leave(&schedule_lock);
526}
527
528static void i915_global_scheduler_shrink(void)
529{
530#ifdef notyet
531 kmem_cache_shrink(global.slab_dependencies);
532 kmem_cache_shrink(global.slab_priorities);
533#endif
534}
535
536static void i915_global_scheduler_exit(void)
537{
538#ifdef __linux__
539 kmem_cache_destroy(global.slab_dependencies);
540 kmem_cache_destroy(global.slab_priorities);
541#else
542 pool_destroy(&global.slab_dependencies);
543 pool_destroy(&global.slab_priorities);
544#endif
545}
546
547static struct i915_global_scheduler global = { {
548 .shrink = i915_global_scheduler_shrink,
549 .exit = i915_global_scheduler_exit,
550} };
551
552int __init i915_global_scheduler_init(void)
553{
554#ifdef __linux__
555 global.slab_dependencies = KMEM_CACHE(i915_dependency,
556 SLAB_HWCACHE_ALIGN |
557 SLAB_TYPESAFE_BY_RCU);
558 if (!global.slab_dependencies)
559 return -ENOMEM12;
560
561 global.slab_priorities = KMEM_CACHE(i915_priolist,
562 SLAB_HWCACHE_ALIGN);
563 if (!global.slab_priorities)
564 goto err_priorities;
565
566 i915_global_register(&global.base);
567 return 0;
568
569err_priorities:
570 kmem_cache_destroy(global.slab_priorities);
571 return -ENOMEM12;
572#else
573 pool_init(&global.slab_dependencies, sizeof(struct i915_dependency),
574 CACHELINESIZE64, IPL_TTY0x9, 0, "gsdep", NULL((void *)0));
575 pool_init(&global.slab_priorities, sizeof(struct i915_priolist),
576 CACHELINESIZE64, IPL_TTY0x9, 0, "gspri", NULL((void *)0));
577
578 i915_global_register(&global.base);
579 return 0;
580#endif
581}