Bug Summary

File:dev/pci/drm/i915/gt/intel_engine_heartbeat.c
Warning:line 141, column 5
Access to field 'emitted_jiffies' results in a dereference of a null pointer (loaded from variable 'rq')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name intel_engine_heartbeat.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/gt/intel_engine_heartbeat.c

/usr/src/sys/dev/pci/drm/i915/gt/intel_engine_heartbeat.c

→
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include "i915_drv.h"
7#include "i915_request.h"
8
9#include "intel_context.h"
10#include "intel_engine_heartbeat.h"
11#include "intel_engine_pm.h"
12#include "intel_engine.h"
13#include "intel_gt.h"
14#include "intel_reset.h"
15
16/*
17 * While the engine is active, we send a periodic pulse along the engine
18 * to check on its health and to flush any idle-barriers. If that request
19 * is stuck, and we fail to preempt it, we declare the engine hung and
20 * issue a reset -- in the hope that restores progress.
21 */
22
23static bool_Bool next_heartbeat(struct intel_engine_cs *engine)
24{
25 long delay;
26
27 delay = READ_ONCE(engine->props.heartbeat_interval_ms)({ typeof(engine->props.heartbeat_interval_ms) __tmp = *(volatile
typeof(engine->props.heartbeat_interval_ms) *)&(engine
->props.heartbeat_interval_ms); membar_datadep_consumer();
__tmp; });
28 if (!delay)
29 return false0;
30
31 delay = msecs_to_jiffies_timeout(delay);
32 if (delay >= HZhz)
33 delay = round_jiffies_up_relative(delay);
34 mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
35
36 return true1;
37}
38
39static struct i915_request *
40heartbeat_create(struct intel_context *ce, gfp_t gfp)
41{
42 struct i915_request *rq;
43
44 intel_context_enter(ce);
45 rq = __i915_request_create(ce, gfp);
46 intel_context_exit(ce);
47
48 return rq;
49}
50
51static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
52{
53 engine->wakeref_serial = READ_ONCE(engine->serial)({ typeof(engine->serial) __tmp = *(volatile typeof(engine
->serial) *)&(engine->serial); membar_datadep_consumer
(); __tmp; }) + 1;
54 i915_request_add_active_barriers(rq);
55 if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
56 engine->heartbeat.systole = i915_request_get(rq);
57}
58
59static void heartbeat_commit(struct i915_request *rq,
60 const struct i915_sched_attr *attr)
61{
62 idle_pulse(rq->engine, rq);
63
64 __i915_request_commit(rq);
65 __i915_request_queue(rq, attr);
66}
67
68static void show_heartbeat(const struct i915_request *rq,
69 struct intel_engine_cs *engine)
70{
71 struct drm_printer p = drm_debug_printer("heartbeat");
72
73 if (!rq) {
74 intel_engine_dump(engine, &p,
75 "%s heartbeat not ticking\n",
76 engine->name);
77 } else {
78 intel_engine_dump(engine, &p,
79 "%s heartbeat {seqno:%llx:%lld, prio:%d} not ticking\n",
80 engine->name,
81 rq->fence.context,
82 rq->fence.seqno,
83 rq->sched.attr.priority);
84 }
85}
86
87static void
88reset_engine(struct intel_engine_cs *engine, struct i915_request *rq)
89{
90 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)0)
91 show_heartbeat(rq, engine);
92
93 if (intel_engine_uses_guc(engine))
94 /*
95 * GuC itself is toast or GuC's hang detection
96 * is disabled. Either way, need to find the
97 * hang culprit manually.
98 */
99 intel_guc_find_hung_context(engine);
100
101 intel_gt_handle_error(engine->gt, engine->mask,
102 I915_ERROR_CAPTURE(1UL << (0)),
103 "stopped heartbeat on %s",
104 engine->name);
105}
106
107static void heartbeat(struct work_struct *wrk)
108{
109 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
110 struct intel_engine_cs *engine =
111 container_of(wrk, typeof(*engine), heartbeat.work.work)({ const __typeof( ((typeof(*engine) *)0)->heartbeat.work.
work ) *__mptr = (wrk); (typeof(*engine) *)( (char *)__mptr -
__builtin_offsetof(typeof(*engine), heartbeat.work.work) );}
);
112 struct intel_context *ce = engine->kernel_context;
113 struct i915_request *rq;
114 unsigned long serial;
115
116 /* Just in case everything has gone horribly wrong, give it a kick */
117 intel_engine_flush_submission(engine);
1
Calling 'intel_engine_flush_submission'
→
3
←
Returning from 'intel_engine_flush_submission'
→
118
119 rq = engine->heartbeat.systole;
4
←
Value assigned to 'rq'
→
120 if (rq && i915_request_completed(rq)) {
5
←
Assuming 'rq' is null
→
121 i915_request_put(rq);
122 engine->heartbeat.systole = NULL((void *)0);
123 }
124
125 if (!intel_engine_pm_get_if_awake(engine))
6
←
Calling 'intel_engine_pm_get_if_awake'
→
10
←
Returning from 'intel_engine_pm_get_if_awake'
→
11
←
Assuming the condition is false
→
12
←
Taking false branch
→
126 return;
127
128 if (intel_gt_is_wedged(engine->gt))
13
←
Taking false branch
→
129 goto out;
130
131 if (i915_sched_engine_disabled(engine->sched_engine)) {
14
←
Assuming the condition is false
→
15
←
Taking false branch
→
132 reset_engine(engine, engine->heartbeat.systole);
133 goto out;
134 }
135
136 if (engine->heartbeat.systole) {
16
←
Assuming field 'systole' is non-null
→
17
←
Taking true branch
→
137 long delay = READ_ONCE(engine->props.heartbeat_interval_ms)({ typeof(engine->props.heartbeat_interval_ms) __tmp = *(volatile
typeof(engine->props.heartbeat_interval_ms) *)&(engine
->props.heartbeat_interval_ms); membar_datadep_consumer();
__tmp; });
138
139 /* Safeguard against too-fast worker invocations */
140 if (!time_after(jiffies,
141 rq->emitted_jiffies + msecs_to_jiffies(delay)(((uint64_t)(delay)) * hz / 1000)))
18
←
Access to field 'emitted_jiffies' results in a dereference of a null pointer (loaded from variable 'rq')
142 goto out;
143
144 if (!i915_sw_fence_signaled(&rq->submit)) {
145 /*
146 * Not yet submitted, system is stalled.
147 *
148 * This more often happens for ring submission,
149 * where all contexts are funnelled into a common
150 * ringbuffer. If one context is blocked on an
151 * external fence, not only is it not submitted,
152 * but all other contexts, including the kernel
153 * context are stuck waiting for the signal.
154 */
155 } else if (engine->sched_engine->schedule &&
156 rq->sched.attr.priority < I915_PRIORITY_BARRIER(0x7fffffff - 1)) {
157 /*
158 * Gradually raise the priority of the heartbeat to
159 * give high priority work [which presumably desires
160 * low latency and no jitter] the chance to naturally
161 * complete before being preempted.
162 */
163 attr.priority = 0;
164 if (rq->sched.attr.priority >= attr.priority)
165 attr.priority = I915_PRIORITY_HEARTBEAT;
166 if (rq->sched.attr.priority >= attr.priority)
167 attr.priority = I915_PRIORITY_BARRIER(0x7fffffff - 1);
168
169 local_bh_disable();
170 engine->sched_engine->schedule(rq, &attr);
171 local_bh_enable();
172 } else {
173 reset_engine(engine, rq);
174 }
175
176 rq->emitted_jiffies = jiffies;
177 goto out;
178 }
179
180 serial = READ_ONCE(engine->serial)({ typeof(engine->serial) __tmp = *(volatile typeof(engine
->serial) *)&(engine->serial); membar_datadep_consumer
(); __tmp; });
181 if (engine->wakeref_serial == serial)
182 goto out;
183
184 if (!mutex_trylock(&ce->timeline->mutex)(rw_enter(&ce->timeline->mutex, 0x0001UL | 0x0040UL
) == 0)) {
185 /* Unable to lock the kernel timeline, is the engine stuck? */
186 if (xchg(&engine->heartbeat.blocked, serial)__sync_lock_test_and_set(&engine->heartbeat.blocked, serial
) == serial)
187 intel_gt_handle_error(engine->gt, engine->mask,
188 I915_ERROR_CAPTURE(1UL << (0)),
189 "no heartbeat on %s",
190 engine->name);
191 goto out;
192 }
193
194 rq = heartbeat_create(ce, GFP_NOWAIT0x0002 | __GFP_NOWARN0);
195 if (IS_ERR(rq))
196 goto unlock;
197
198 heartbeat_commit(rq, &attr);
199
200unlock:
201 mutex_unlock(&ce->timeline->mutex)rw_exit_write(&ce->timeline->mutex);
202out:
203 if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
204 i915_request_put(fetch_and_zero(&engine->heartbeat.systole)({ typeof(*&engine->heartbeat.systole) __T = *(&engine
->heartbeat.systole); *(&engine->heartbeat.systole)
= (typeof(*&engine->heartbeat.systole))0; __T; }));
205 intel_engine_pm_put(engine);
206}
207
208void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
209{
210 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL2500)
211 return;
212
213 next_heartbeat(engine);
214}
215
216void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
217{
218 if (cancel_delayed_work(&engine->heartbeat.work))
219 i915_request_put(fetch_and_zero(&engine->heartbeat.systole)({ typeof(*&engine->heartbeat.systole) __T = *(&engine
->heartbeat.systole); *(&engine->heartbeat.systole)
= (typeof(*&engine->heartbeat.systole))0; __T; }));
220}
221
222void intel_gt_unpark_heartbeats(struct intel_gt *gt)
223{
224 struct intel_engine_cs *engine;
225 enum intel_engine_id id;
226
227 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else
228 if (intel_engine_pm_is_awake(engine))
229 intel_engine_unpark_heartbeat(engine);
230}
231
232void intel_gt_park_heartbeats(struct intel_gt *gt)
233{
234 struct intel_engine_cs *engine;
235 enum intel_engine_id id;
236
237 for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else
238 intel_engine_park_heartbeat(engine);
239}
240
241void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
242{
243 INIT_DELAYED_WORK(&engine->heartbeat.work, heartbeat);
244}
245
246static int __intel_engine_pulse(struct intel_engine_cs *engine)
247{
248 struct i915_sched_attr attr = { .priority = I915_PRIORITY_BARRIER(0x7fffffff - 1) };
249 struct intel_context *ce = engine->kernel_context;
250 struct i915_request *rq;
251
252 lockdep_assert_held(&ce->timeline->mutex)do { (void)(&ce->timeline->mutex); } while(0);
253 GEM_BUG_ON(!intel_engine_has_preemption(engine))((void)0);
254 GEM_BUG_ON(!intel_engine_pm_is_awake(engine))((void)0);
255
256 rq = heartbeat_create(ce, GFP_NOWAIT0x0002 | __GFP_NOWARN0);
257 if (IS_ERR(rq))
258 return PTR_ERR(rq);
259
260 __set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
261
262 heartbeat_commit(rq, &attr);
263 GEM_BUG_ON(rq->sched.attr.priority < I915_PRIORITY_BARRIER)((void)0);
264
265 return 0;
266}
267
268static unsigned long set_heartbeat(struct intel_engine_cs *engine,
269 unsigned long delay)
270{
271 unsigned long old;
272
273 old = xchg(&engine->props.heartbeat_interval_ms, delay)__sync_lock_test_and_set(&engine->props.heartbeat_interval_ms
, delay);
274 if (delay)
275 intel_engine_unpark_heartbeat(engine);
276 else
277 intel_engine_park_heartbeat(engine);
278
279 return old;
280}
281
282int intel_engine_set_heartbeat(struct intel_engine_cs *engine,
283 unsigned long delay)
284{
285 struct intel_context *ce = engine->kernel_context;
286 int err = 0;
287
288 if (!delay && !intel_engine_has_preempt_reset(engine))
289 return -ENODEV19;
290
291 intel_engine_pm_get(engine);
292
293 err = mutex_lock_interruptible(&ce->timeline->mutex);
294 if (err)
295 goto out_rpm;
296
297 if (delay != engine->props.heartbeat_interval_ms) {
298 unsigned long saved = set_heartbeat(engine, delay);
299
300 /* recheck current execution */
301 if (intel_engine_has_preemption(engine)) {
302 err = __intel_engine_pulse(engine);
303 if (err)
304 set_heartbeat(engine, saved);
305 }
306 }
307
308 mutex_unlock(&ce->timeline->mutex)rw_exit_write(&ce->timeline->mutex);
309
310out_rpm:
311 intel_engine_pm_put(engine);
312 return err;
313}
314
315int intel_engine_pulse(struct intel_engine_cs *engine)
316{
317 struct intel_context *ce = engine->kernel_context;
318 int err;
319
320 if (!intel_engine_has_preemption(engine))
321 return -ENODEV19;
322
323 if (!intel_engine_pm_get_if_awake(engine))
324 return 0;
325
326 err = -EINTR4;
327 if (!mutex_lock_interruptible(&ce->timeline->mutex)) {
328 err = __intel_engine_pulse(engine);
329 mutex_unlock(&ce->timeline->mutex)rw_exit_write(&ce->timeline->mutex);
330 }
331
332 intel_engine_flush_submission(engine);
333 intel_engine_pm_put(engine);
334 return err;
335}
336
337int intel_engine_flush_barriers(struct intel_engine_cs *engine)
338{
339 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
340 struct intel_context *ce = engine->kernel_context;
341 struct i915_request *rq;
342 int err;
343
344 if (llist_empty(&engine->barrier_tasks))
345 return 0;
346
347 if (!intel_engine_pm_get_if_awake(engine))
348 return 0;
349
350 if (mutex_lock_interruptible(&ce->timeline->mutex)) {
351 err = -EINTR4;
352 goto out_rpm;
353 }
354
355 rq = heartbeat_create(ce, GFP_KERNEL(0x0001 | 0x0004));
356 if (IS_ERR(rq)) {
357 err = PTR_ERR(rq);
358 goto out_unlock;
359 }
360
361 heartbeat_commit(rq, &attr);
362
363 err = 0;
364out_unlock:
365 mutex_unlock(&ce->timeline->mutex)rw_exit_write(&ce->timeline->mutex);
366out_rpm:
367 intel_engine_pm_put(engine);
368 return err;
369}
370
371#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
372#include "selftest_engine_heartbeat.c"
373#endif
←

/usr/src/sys/dev/pci/drm/i915/gt/intel_engine.h

→
1/* SPDX-License-Identifier: MIT */
2#ifndef _INTEL_RINGBUFFER_H_
3#define _INTEL_RINGBUFFER_H_
4
5#include <asm/cacheflush.h>
6#include <drm/drm_util.h>
7#include <drm/drm_cache.h>
8
9#include <linux/hashtable.h>
10#include <linux/irq_work.h>
11#include <linux/random.h>
12#include <linux/seqlock.h>
13
14#include "i915_pmu.h"
15#include "i915_request.h"
16#include "i915_selftest.h"
17#include "intel_engine_types.h"
18#include "intel_gt_types.h"
19#include "intel_timeline.h"
20#include "intel_workarounds.h"
21
22struct drm_printer;
23struct intel_context;
24struct intel_gt;
25struct lock_class_key;
26
27/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
28 * but keeps the logic simple. Indeed, the whole purpose of this macro is just
29 * to give some inclination as to some of the magic values used in the various
30 * workarounds!
31 */
32#define CACHELINE_BYTES64 64
33#define CACHELINE_DWORDS(64 / sizeof(u32)) (CACHELINE_BYTES64 / sizeof(u32))
34
35#define ENGINE_TRACE(e, fmt, ...)do { const struct intel_engine_cs *e__ __attribute__((__unused__
)) = (e); do { } while (0); } while (0) do { \
36 const struct intel_engine_cs *e__ __maybe_unused__attribute__((__unused__)) = (e); \
37 GEM_TRACE("%s %s: " fmt, \do { } while (0)
38 dev_name(e__->i915->drm.dev), e__->name, \do { } while (0)
39 ##__VA_ARGS__)do { } while (0); \
40} while (0)
41
42/*
43 * The register defines to be used with the following macros need to accept a
44 * base param, e.g:
45 *
46 * REG_FOO(base) _MMIO((base) + <relative offset>)
47 * ENGINE_READ(engine, REG_FOO);
48 *
49 * register arrays are to be defined and accessed as follows:
50 *
51 * REG_BAR(base, i) _MMIO((base) + <relative offset> + (i) * <shift>)
52 * ENGINE_READ_IDX(engine, REG_BAR, i)
53 */
54
55#define __ENGINE_REG_OP(op__, engine__, ...)intel_uncore_op__((engine__)->uncore, ...) \
56 intel_uncore_##op__((engine__)->uncore, __VA_ARGS__)
57
58#define __ENGINE_READ_OP(op__, engine__, reg__)intel_uncore_op__(((engine__))->uncore, reg__((engine__)->
mmio_base)) \
59 __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base))intel_uncore_op__(((engine__))->uncore, reg__((engine__)->
mmio_base))
60
61#define ENGINE_READ16(...)__ENGINE_READ_OP __ENGINE_READ_OP(read16, __VA_ARGS__)
62#define ENGINE_READ(...)__ENGINE_READ_OP __ENGINE_READ_OP(read, __VA_ARGS__)
63#define ENGINE_READ_FW(...)__ENGINE_READ_OP __ENGINE_READ_OP(read_fw, __VA_ARGS__)
64#define ENGINE_POSTING_READ(...)__ENGINE_READ_OP __ENGINE_READ_OP(posting_read_fw, __VA_ARGS__)
65#define ENGINE_POSTING_READ16(...)__ENGINE_READ_OP __ENGINE_READ_OP(posting_read16, __VA_ARGS__)
66
67#define ENGINE_READ64(engine__, lower_reg__, upper_reg__)intel_uncore_read64_2x32(((engine__))->uncore, lower_reg__
((engine__)->mmio_base), upper_reg__((engine__)->mmio_base
)) \
68 __ENGINE_REG_OP(read64_2x32, (engine__), \intel_uncore_read64_2x32(((engine__))->uncore, lower_reg__
((engine__)->mmio_base), upper_reg__((engine__)->mmio_base
))
69 lower_reg__((engine__)->mmio_base), \intel_uncore_read64_2x32(((engine__))->uncore, lower_reg__
((engine__)->mmio_base), upper_reg__((engine__)->mmio_base
))
70 upper_reg__((engine__)->mmio_base))intel_uncore_read64_2x32(((engine__))->uncore, lower_reg__
((engine__)->mmio_base), upper_reg__((engine__)->mmio_base
))
71
72#define ENGINE_READ_IDX(engine__, reg__, idx__)intel_uncore_read(((engine__))->uncore, reg__((engine__)->
mmio_base, (idx__))) \
73 __ENGINE_REG_OP(read, (engine__), reg__((engine__)->mmio_base, (idx__)))intel_uncore_read(((engine__))->uncore, reg__((engine__)->
mmio_base, (idx__)))
74
75#define __ENGINE_WRITE_OP(op__, engine__, reg__, val__)intel_uncore_op__(((engine__))->uncore, reg__((engine__)->
mmio_base), (val__)) \
76 __ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base), (val__))intel_uncore_op__(((engine__))->uncore, reg__((engine__)->
mmio_base), (val__))
77
78#define ENGINE_WRITE16(...)__ENGINE_WRITE_OP __ENGINE_WRITE_OP(write16, __VA_ARGS__)
79#define ENGINE_WRITE(...)__ENGINE_WRITE_OP __ENGINE_WRITE_OP(write, __VA_ARGS__)
80#define ENGINE_WRITE_FW(...)__ENGINE_WRITE_OP __ENGINE_WRITE_OP(write_fw, __VA_ARGS__)
81
82#define GEN6_RING_FAULT_REG_READ(engine__)intel_uncore_read((engine__)->uncore, RING_FAULT_REG(engine__
)) \
83 intel_uncore_read((engine__)->uncore, RING_FAULT_REG(engine__))
84
85#define GEN6_RING_FAULT_REG_POSTING_READ(engine__)((void)intel_uncore_read_notrace((engine__)->uncore, RING_FAULT_REG
(engine__))) \
86 intel_uncore_posting_read((engine__)->uncore, RING_FAULT_REG(engine__))((void)intel_uncore_read_notrace((engine__)->uncore, RING_FAULT_REG
(engine__)))
87
88#define GEN6_RING_FAULT_REG_RMW(engine__, clear__, set__)({ u32 __val; __val = intel_uncore_read((engine__)->uncore
, RING_FAULT_REG(engine__)); __val &= ~(clear__); __val |=
(set__); intel_uncore_write((engine__)->uncore, RING_FAULT_REG
(engine__), __val); }) \
89({ \
90 u32 __val; \
91\
92 __val = intel_uncore_read((engine__)->uncore, \
93 RING_FAULT_REG(engine__)); \
94 __val &= ~(clear__); \
95 __val |= (set__); \
96 intel_uncore_write((engine__)->uncore, RING_FAULT_REG(engine__), \
97 __val); \
98})
99
100/* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
101 * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
102 */
103
104static inline unsigned int
105execlists_num_ports(const struct intel_engine_execlists * const execlists)
106{
107 return execlists->port_mask + 1;
108}
109
110static inline struct i915_request *
111execlists_active(const struct intel_engine_execlists *execlists)
112{
113 struct i915_request * const *cur, * const *old, *active;
114
115 cur = READ_ONCE(execlists->active)({ typeof(execlists->active) __tmp = *(volatile typeof(execlists
->active) *)&(execlists->active); membar_datadep_consumer
(); __tmp; });
116 smp_rmb()do { __asm volatile("" ::: "memory"); } while (0); /* pairs with overwrite protection in process_csb() */
117 do {
118 old = cur;
119
120 active = READ_ONCE(*cur)({ typeof(*cur) __tmp = *(volatile typeof(*cur) *)&(*cur)
; membar_datadep_consumer(); __tmp; });
121 cur = READ_ONCE(execlists->active)({ typeof(execlists->active) __tmp = *(volatile typeof(execlists
->active) *)&(execlists->active); membar_datadep_consumer
(); __tmp; });
122
123 smp_rmb()do { __asm volatile("" ::: "memory"); } while (0); /* and complete the seqlock retry */
124 } while (unlikely(cur != old)__builtin_expect(!!(cur != old), 0));
125
126 return active;
127}
128
129struct i915_request *
130execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists);
131
132static inline u32
133intel_read_status_page(const struct intel_engine_cs *engine, int reg)
134{
135 /* Ensure that the compiler doesn't optimize away the load. */
136 return READ_ONCE(engine->status_page.addr[reg])({ typeof(engine->status_page.addr[reg]) __tmp = *(volatile
typeof(engine->status_page.addr[reg]) *)&(engine->
status_page.addr[reg]); membar_datadep_consumer(); __tmp; });
137}
138
139static inline void
140intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
141{
142 /* Writing into the status page should be done sparingly. Since
143 * we do when we are uncertain of the device state, we take a bit
144 * of extra paranoia to try and ensure that the HWS takes the value
145 * we give and that it doesn't end up trapped inside the CPU!
146 */
147 drm_clflush_virt_range(&engine->status_page.addr[reg], sizeof(value));
148 WRITE_ONCE(engine->status_page.addr[reg], value)({ typeof(engine->status_page.addr[reg]) __tmp = (value); *
(volatile typeof(engine->status_page.addr[reg]) *)&(engine
->status_page.addr[reg]) = __tmp; __tmp; });
149 drm_clflush_virt_range(&engine->status_page.addr[reg], sizeof(value));
150}
151
152/*
153 * Reads a dword out of the status page, which is written to from the command
154 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
155 * MI_STORE_DATA_IMM.
156 *
157 * The following dwords have a reserved meaning:
158 * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
159 * 0x04: ring 0 head pointer
160 * 0x05: ring 1 head pointer (915-class)
161 * 0x06: ring 2 head pointer (915-class)
162 * 0x10-0x1b: Context status DWords (GM45)
163 * 0x1f: Last written status offset. (GM45)
164 * 0x20-0x2f: Reserved (Gen6+)
165 *
166 * The area from dword 0x30 to 0x3ff is available for driver usage.
167 */
168#define I915_GEM_HWS_PREEMPT0x32 0x32
169#define I915_GEM_HWS_PREEMPT_ADDR(0x32 * sizeof(u32)) (I915_GEM_HWS_PREEMPT0x32 * sizeof(u32))
170#define I915_GEM_HWS_SEQNO0x40 0x40
171#define I915_GEM_HWS_SEQNO_ADDR(0x40 * sizeof(u32)) (I915_GEM_HWS_SEQNO0x40 * sizeof(u32))
172#define I915_GEM_HWS_MIGRATE(0x42 * sizeof(u32)) (0x42 * sizeof(u32))
173#define I915_GEM_HWS_PXP0x60 0x60
174#define I915_GEM_HWS_PXP_ADDR(0x60 * sizeof(u32)) (I915_GEM_HWS_PXP0x60 * sizeof(u32))
175#define I915_GEM_HWS_SCRATCH0x80 0x80
176
177#define I915_HWS_CSB_BUF0_INDEX0x10 0x10
178#define I915_HWS_CSB_WRITE_INDEX0x1f 0x1f
179#define ICL_HWS_CSB_WRITE_INDEX0x2f 0x2f
180#define INTEL_HWS_CSB_WRITE_INDEX(__i915)(((&(__i915)->__runtime)->graphics.ip.ver) >= 11
? 0x2f : 0x1f) \
181 (GRAPHICS_VER(__i915)((&(__i915)->__runtime)->graphics.ip.ver) >= 11 ? ICL_HWS_CSB_WRITE_INDEX0x2f : I915_HWS_CSB_WRITE_INDEX0x1f)
182
183void intel_engine_stop(struct intel_engine_cs *engine);
184void intel_engine_cleanup(struct intel_engine_cs *engine);
185
186int intel_engines_init_mmio(struct intel_gt *gt);
187int intel_engines_init(struct intel_gt *gt);
188
189void intel_engine_free_request_pool(struct intel_engine_cs *engine);
190
191void intel_engines_release(struct intel_gt *gt);
192void intel_engines_free(struct intel_gt *gt);
193
194int intel_engine_init_common(struct intel_engine_cs *engine);
195void intel_engine_cleanup_common(struct intel_engine_cs *engine);
196
197int intel_engine_resume(struct intel_engine_cs *engine);
198
199int intel_ring_submission_setup(struct intel_engine_cs *engine);
200
201int intel_engine_stop_cs(struct intel_engine_cs *engine);
202void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine);
203
204void intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs *engine);
205
206void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask);
207
208u64 intel_engine_get_active_head(const struct intel_engine_cs *engine);
209u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine);
210
211void intel_engine_get_instdone(const struct intel_engine_cs *engine,
212 struct intel_instdone *instdone);
213
214void intel_engine_init_execlists(struct intel_engine_cs *engine);
215
216bool_Bool intel_engine_irq_enable(struct intel_engine_cs *engine);
217void intel_engine_irq_disable(struct intel_engine_cs *engine);
218
219static inline void __intel_engine_reset(struct intel_engine_cs *engine,
220 bool_Bool stalled)
221{
222 if (engine->reset.rewind)
223 engine->reset.rewind(engine, stalled);
224 engine->serial++; /* contexts lost */
225}
226
227bool_Bool intel_engines_are_idle(struct intel_gt *gt);
228bool_Bool intel_engine_is_idle(struct intel_engine_cs *engine);
229
230void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool_Bool sync);
231static inline void intel_engine_flush_submission(struct intel_engine_cs *engine)
232{
233 __intel_engine_flush_submission(engine, true1);
2
←
Value assigned to field 'systole'
→
234}
235
236void intel_engines_reset_default_submission(struct intel_gt *gt);
237
238bool_Bool intel_engine_can_store_dword(struct intel_engine_cs *engine);
239
240__printf(3, 4)__attribute__((__format__(__kprintf__,3,4)))
241void intel_engine_dump(struct intel_engine_cs *engine,
242 struct drm_printer *m,
243 const char *header, ...);
244void intel_engine_dump_active_requests(struct list_head *requests,
245 struct i915_request *hung_rq,
246 struct drm_printer *m);
247
248ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine,
249 ktime_t *now);
250
251void intel_engine_get_hung_entity(struct intel_engine_cs *engine,
252 struct intel_context **ce, struct i915_request **rq);
253
254u32 intel_engine_context_size(struct intel_gt *gt, u8 class);
255struct intel_context *
256intel_engine_create_pinned_context(struct intel_engine_cs *engine,
257 struct i915_address_space *vm,
258 unsigned int ring_size,
259 unsigned int hwsp,
260 struct lock_class_key *key,
261 const char *name);
262
263void intel_engine_destroy_pinned_context(struct intel_context *ce);
264
265void xehp_enable_ccs_engines(struct intel_engine_cs *engine);
266
267#define ENGINE_PHYSICAL0 0
268#define ENGINE_MOCK1 1
269#define ENGINE_VIRTUAL2 2
270
271static inline bool_Bool intel_engine_uses_guc(const struct intel_engine_cs *engine)
272{
273 return engine->gt->submission_method >= INTEL_SUBMISSION_GUC;
274}
275
276static inline bool_Bool
277intel_engine_has_preempt_reset(const struct intel_engine_cs *engine)
278{
279 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT640)
280 return false0;
281
282 return intel_engine_has_preemption(engine);
283}
284
285#define FORCE_VIRTUAL(1UL << (0)) BIT(0)(1UL << (0))
286struct intel_context *
287intel_engine_create_virtual(struct intel_engine_cs **siblings,
288 unsigned int count, unsigned long flags);
289
290static inline struct intel_context *
291intel_engine_create_parallel(struct intel_engine_cs **engines,
292 unsigned int num_engines,
293 unsigned int width)
294{
295 GEM_BUG_ON(!engines[0]->cops->create_parallel)((void)0);
296 return engines[0]->cops->create_parallel(engines, num_engines, width);
297}
298
299static inline bool_Bool
300intel_virtual_engine_has_heartbeat(const struct intel_engine_cs *engine)
301{
302 /*
303 * For non-GuC submission we expect the back-end to look at the
304 * heartbeat status of the actual physical engine that the work
305 * has been (or is being) scheduled on, so we should only reach
306 * here with GuC submission enabled.
307 */
308 GEM_BUG_ON(!intel_engine_uses_guc(engine))((void)0);
309
310 return intel_guc_virtual_engine_has_heartbeat(engine);
311}
312
313static inline bool_Bool
314intel_engine_has_heartbeat(const struct intel_engine_cs *engine)
315{
316 if (!CONFIG_DRM_I915_HEARTBEAT_INTERVAL2500)
317 return false0;
318
319 if (intel_engine_is_virtual(engine))
320 return intel_virtual_engine_has_heartbeat(engine);
321 else
322 return READ_ONCE(engine->props.heartbeat_interval_ms)({ typeof(engine->props.heartbeat_interval_ms) __tmp = *(volatile
typeof(engine->props.heartbeat_interval_ms) *)&(engine
->props.heartbeat_interval_ms); membar_datadep_consumer();
__tmp; });
323}
324
325static inline struct intel_engine_cs *
326intel_engine_get_sibling(struct intel_engine_cs *engine, unsigned int sibling)
327{
328 GEM_BUG_ON(!intel_engine_is_virtual(engine))((void)0);
329 return engine->cops->get_sibling(engine, sibling);
330}
331
332static inline void
333intel_engine_set_hung_context(struct intel_engine_cs *engine,
334 struct intel_context *ce)
335{
336 engine->hung_ce = ce;
337}
338
339static inline void
340intel_engine_clear_hung_context(struct intel_engine_cs *engine)
341{
342 intel_engine_set_hung_context(engine, NULL((void *)0));
343}
344
345static inline struct intel_context *
346intel_engine_get_hung_context(struct intel_engine_cs *engine)
347{
348 return engine->hung_ce;
349}
350
351u64 intel_clamp_heartbeat_interval_ms(struct intel_engine_cs *engine, u64 value);
352u64 intel_clamp_max_busywait_duration_ns(struct intel_engine_cs *engine, u64 value);
353u64 intel_clamp_preempt_timeout_ms(struct intel_engine_cs *engine, u64 value);
354u64 intel_clamp_stop_timeout_ms(struct intel_engine_cs *engine, u64 value);
355u64 intel_clamp_timeslice_duration_ms(struct intel_engine_cs *engine, u64 value);
356
357#endif /* _INTEL_RINGBUFFER_H_ */
←

/usr/src/sys/dev/pci/drm/i915/gt/intel_engine_pm.h

→
1/* SPDX-License-Identifier: MIT */
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#ifndef INTEL_ENGINE_PM_H
7#define INTEL_ENGINE_PM_H
8
9#include "i915_drv.h"
10#include "i915_request.h"
11#include "intel_engine_types.h"
12#include "intel_wakeref.h"
13#include "intel_gt_pm.h"
14
15static inline bool_Bool
16intel_engine_pm_is_awake(const struct intel_engine_cs *engine)
17{
18 return intel_wakeref_is_active(&engine->wakeref);
19}
20
21static inline void __intel_engine_pm_get(struct intel_engine_cs *engine)
22{
23 __intel_wakeref_get(&engine->wakeref);
24}
25
26static inline void intel_engine_pm_get(struct intel_engine_cs *engine)
27{
28 intel_wakeref_get(&engine->wakeref);
29}
30
31static inline bool_Bool intel_engine_pm_get_if_awake(struct intel_engine_cs *engine)
32{
33 return intel_wakeref_get_if_active(&engine->wakeref);
7
←
Calling 'intel_wakeref_get_if_active'
→
9
←
Returning from 'intel_wakeref_get_if_active'
→
34}
35
36static inline void intel_engine_pm_might_get(struct intel_engine_cs *engine)
37{
38 if (!intel_engine_is_virtual(engine)) {
39 intel_wakeref_might_get(&engine->wakeref);
40 } else {
41 struct intel_gt *gt = engine->gt;
42 struct intel_engine_cs *tengine;
43 intel_engine_mask_t tmp, mask = engine->mask;
44
45 for_each_engine_masked(tengine, gt, mask, tmp)for ((tmp) = (mask) & (gt)->info.engine_mask; (tmp) ? (
(tengine) = (gt)->engine[({ int __idx = ffs(tmp) - 1; tmp &=
~(1UL << (__idx)); __idx; })]), 1 : 0;)
46 intel_wakeref_might_get(&tengine->wakeref);
47 }
48 intel_gt_pm_might_get(engine->gt);
49}
50
51static inline void intel_engine_pm_put(struct intel_engine_cs *engine)
52{
53 intel_wakeref_put(&engine->wakeref);
54}
55
56static inline void intel_engine_pm_put_async(struct intel_engine_cs *engine)
57{
58 intel_wakeref_put_async(&engine->wakeref);
59}
60
61static inline void intel_engine_pm_put_delay(struct intel_engine_cs *engine,
62 unsigned long delay)
63{
64 intel_wakeref_put_delay(&engine->wakeref, delay);
65}
66
67static inline void intel_engine_pm_flush(struct intel_engine_cs *engine)
68{
69 intel_wakeref_unlock_wait(&engine->wakeref);
70}
71
72static inline void intel_engine_pm_might_put(struct intel_engine_cs *engine)
73{
74 if (!intel_engine_is_virtual(engine)) {
75 intel_wakeref_might_put(&engine->wakeref);
76 } else {
77 struct intel_gt *gt = engine->gt;
78 struct intel_engine_cs *tengine;
79 intel_engine_mask_t tmp, mask = engine->mask;
80
81 for_each_engine_masked(tengine, gt, mask, tmp)for ((tmp) = (mask) & (gt)->info.engine_mask; (tmp) ? (
(tengine) = (gt)->engine[({ int __idx = ffs(tmp) - 1; tmp &=
~(1UL << (__idx)); __idx; })]), 1 : 0;)
82 intel_wakeref_might_put(&tengine->wakeref);
83 }
84 intel_gt_pm_might_put(engine->gt);
85}
86
87static inline struct i915_request *
88intel_engine_create_kernel_request(struct intel_engine_cs *engine)
89{
90 struct i915_request *rq;
91
92 /*
93 * The engine->kernel_context is special as it is used inside
94 * the engine-pm barrier (see __engine_park()), circumventing
95 * the usual mutexes and relying on the engine-pm barrier
96 * instead. So whenever we use the engine->kernel_context
97 * outside of the barrier, we must manually handle the
98 * engine wakeref to serialise with the use inside.
99 */
100 intel_engine_pm_get(engine);
101 rq = i915_request_create(engine->kernel_context);
102 intel_engine_pm_put(engine);
103
104 return rq;
105}
106
107void intel_engine_init__pm(struct intel_engine_cs *engine);
108
109void intel_engine_reset_pinned_contexts(struct intel_engine_cs *engine);
110
111#endif /* INTEL_ENGINE_PM_H */
←

/usr/src/sys/dev/pci/drm/i915/intel_wakeref.h

1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7#ifndef INTEL_WAKEREF_H
8#define INTEL_WAKEREF_H
9
10#include <linux/atomic.h>
11#include <linux/bitfield.h>
12#include <linux/bits.h>
13#include <linux/lockdep.h>
14#include <linux/mutex.h>
15#include <linux/refcount.h>
16#include <linux/stackdepot.h>
17#include <linux/timer.h>
18#include <linux/workqueue.h>
19
20#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)0
21#define INTEL_WAKEREF_BUG_ON(expr)((void)0) BUG_ON(expr)((!(expr)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/i915/intel_wakeref.h"
, 21, "!(expr)"))
22#else
23#define INTEL_WAKEREF_BUG_ON(expr)((void)0) BUILD_BUG_ON_INVALID(expr)((void)0)
24#endif
25
26struct intel_runtime_pm;
27struct intel_wakeref;
28
29typedef depot_stack_handle_t intel_wakeref_t;
30
31struct intel_wakeref_ops {
32 int (*get)(struct intel_wakeref *wf);
33 int (*put)(struct intel_wakeref *wf);
34};
35
36struct intel_wakeref {
37 atomic_t count;
38 struct rwlock mutex;
39
40 intel_wakeref_t wakeref;
41
42 struct intel_runtime_pm *rpm;
43 const struct intel_wakeref_ops *ops;
44
45 struct delayed_work work;
46};
47
48struct intel_wakeref_lockclass {
49 struct lock_class_key mutex;
50 struct lock_class_key work;
51};
52
53void __intel_wakeref_init(struct intel_wakeref *wf,
54 struct intel_runtime_pm *rpm,
55 const struct intel_wakeref_ops *ops,
56 struct intel_wakeref_lockclass *key);
57#define intel_wakeref_init(wf, rpm, ops)do { static struct intel_wakeref_lockclass __key; __intel_wakeref_init
((wf), (rpm), (ops), &__key); } while (0) do { \
58 static struct intel_wakeref_lockclass __key; \
59 \
60 __intel_wakeref_init((wf), (rpm), (ops), &__key); \
61} while (0)
62
63int __intel_wakeref_get_first(struct intel_wakeref *wf);
64void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags);
65
66/**
67 * intel_wakeref_get: Acquire the wakeref
68 * @wf: the wakeref
69 *
70 * Acquire a hold on the wakeref. The first user to do so, will acquire
71 * the runtime pm wakeref and then call the @fn underneath the wakeref
72 * mutex.
73 *
74 * Note that @fn is allowed to fail, in which case the runtime-pm wakeref
75 * will be released and the acquisition unwound, and an error reported.
76 *
77 * Returns: 0 if the wakeref was acquired successfully, or a negative error
78 * code otherwise.
79 */
80static inline int
81intel_wakeref_get(struct intel_wakeref *wf)
82{
83 might_sleep()assertwaitok();
84 if (unlikely(!atomic_inc_not_zero(&wf->count))__builtin_expect(!!(!atomic_add_unless((&wf->count), 1
, 0)), 0))
85 return __intel_wakeref_get_first(wf);
86
87 return 0;
88}
89
90/**
91 * __intel_wakeref_get: Acquire the wakeref, again
92 * @wf: the wakeref
93 *
94 * Increment the wakeref counter, only valid if it is already held by
95 * the caller.
96 *
97 * See intel_wakeref_get().
98 */
99static inline void
100__intel_wakeref_get(struct intel_wakeref *wf)
101{
102 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0)((void)0);
103 atomic_inc(&wf->count)__sync_fetch_and_add(&wf->count, 1);
104}
105
106/**
107 * intel_wakeref_get_if_in_use: Acquire the wakeref
108 * @wf: the wakeref
109 *
110 * Acquire a hold on the wakeref, but only if the wakeref is already
111 * active.
112 *
113 * Returns: true if the wakeref was acquired, false otherwise.
114 */
115static inline bool_Bool
116intel_wakeref_get_if_active(struct intel_wakeref *wf)
117{
118 return atomic_inc_not_zero(&wf->count)atomic_add_unless((&wf->count), 1, 0);
8
←
Value assigned to field 'systole', which participates in a condition later
→
119}
120
121enum {
122 INTEL_WAKEREF_PUT_ASYNC_BIT = 0,
123 __INTEL_WAKEREF_PUT_LAST_BIT__
124};
125
126static inline void
127intel_wakeref_might_get(struct intel_wakeref *wf)
128{
129 might_lock(&wf->mutex);
130}
131
132/**
133 * intel_wakeref_put_flags: Release the wakeref
134 * @wf: the wakeref
135 * @flags: control flags
136 *
137 * Release our hold on the wakeref. When there are no more users,
138 * the runtime pm wakeref will be released after the @fn callback is called
139 * underneath the wakeref mutex.
140 *
141 * Note that @fn is allowed to fail, in which case the runtime-pm wakeref
142 * is retained and an error reported.
143 *
144 * Returns: 0 if the wakeref was released successfully, or a negative error
145 * code otherwise.
146 */
147static inline void
148__intel_wakeref_put(struct intel_wakeref *wf, unsigned long flags)
149#define INTEL_WAKEREF_PUT_ASYNC(1UL << (INTEL_WAKEREF_PUT_ASYNC_BIT)) BIT(INTEL_WAKEREF_PUT_ASYNC_BIT)(1UL << (INTEL_WAKEREF_PUT_ASYNC_BIT))
150#define INTEL_WAKEREF_PUT_DELAY(((~0UL) >> (64 - (64 - 1) - 1)) & ((~0UL) <<
(__INTEL_WAKEREF_PUT_LAST_BIT__))) \
151 GENMASK(BITS_PER_LONG - 1, __INTEL_WAKEREF_PUT_LAST_BIT__)(((~0UL) >> (64 - (64 - 1) - 1)) & ((~0UL) <<
(__INTEL_WAKEREF_PUT_LAST_BIT__)))
152{
153 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0)((void)0);
154 if (unlikely(!atomic_add_unless(&wf->count, -1, 1))__builtin_expect(!!(!atomic_add_unless(&wf->count, -1,
1)), 0))
155 __intel_wakeref_put_last(wf, flags);
156}
157
158static inline void
159intel_wakeref_put(struct intel_wakeref *wf)
160{
161 might_sleep()assertwaitok();
162 __intel_wakeref_put(wf, 0);
163}
164
165static inline void
166intel_wakeref_put_async(struct intel_wakeref *wf)
167{
168 __intel_wakeref_put(wf, INTEL_WAKEREF_PUT_ASYNC(1UL << (INTEL_WAKEREF_PUT_ASYNC_BIT)));
169}
170
171static inline void
172intel_wakeref_put_delay(struct intel_wakeref *wf, unsigned long delay)
173{
174 __intel_wakeref_put(wf,
175 INTEL_WAKEREF_PUT_ASYNC(1UL << (INTEL_WAKEREF_PUT_ASYNC_BIT)) |
176 FIELD_PREP(INTEL_WAKEREF_PUT_DELAY, delay)(((typeof((((~0UL) >> (64 - (64 - 1) - 1)) & ((~0UL
) << (__INTEL_WAKEREF_PUT_LAST_BIT__)))))(delay) <<
(__builtin_ffsll((((~0UL) >> (64 - (64 - 1) - 1)) &
((~0UL) << (__INTEL_WAKEREF_PUT_LAST_BIT__)))) - 1)) &
((((~0UL) >> (64 - (64 - 1) - 1)) & ((~0UL) <<
(__INTEL_WAKEREF_PUT_LAST_BIT__))))));
177}
178
179static inline void
180intel_wakeref_might_put(struct intel_wakeref *wf)
181{
182 might_lock(&wf->mutex);
183}
184
185/**
186 * intel_wakeref_lock: Lock the wakeref (mutex)
187 * @wf: the wakeref
188 *
189 * Locks the wakeref to prevent it being acquired or released. New users
190 * can still adjust the counter, but the wakeref itself (and callback)
191 * cannot be acquired or released.
192 */
193static inline void
194intel_wakeref_lock(struct intel_wakeref *wf)
195 __acquires(wf->mutex)
196{
197 mutex_lock(&wf->mutex)rw_enter_write(&wf->mutex);
198}
199
200/**
201 * intel_wakeref_unlock: Unlock the wakeref
202 * @wf: the wakeref
203 *
204 * Releases a previously acquired intel_wakeref_lock().
205 */
206static inline void
207intel_wakeref_unlock(struct intel_wakeref *wf)
208 __releases(wf->mutex)
209{
210 mutex_unlock(&wf->mutex)rw_exit_write(&wf->mutex);
211}
212
213/**
214 * intel_wakeref_unlock_wait: Wait until the active callback is complete
215 * @wf: the wakeref
216 *
217 * Waits for the active callback (under the @wf->mutex or another CPU) is
218 * complete.
219 */
220static inline void
221intel_wakeref_unlock_wait(struct intel_wakeref *wf)
222{
223 mutex_lock(&wf->mutex)rw_enter_write(&wf->mutex);
224 mutex_unlock(&wf->mutex)rw_exit_write(&wf->mutex);
225 flush_delayed_work(&wf->work);
226}
227
228/**
229 * intel_wakeref_is_active: Query whether the wakeref is currently held
230 * @wf: the wakeref
231 *
232 * Returns: true if the wakeref is currently held.
233 */
234static inline bool_Bool
235intel_wakeref_is_active(const struct intel_wakeref *wf)
236{
237 return READ_ONCE(wf->wakeref)({ typeof(wf->wakeref) __tmp = *(volatile typeof(wf->wakeref
) *)&(wf->wakeref); membar_datadep_consumer(); __tmp; }
);
238}
239
240/**
241 * __intel_wakeref_defer_park: Defer the current park callback
242 * @wf: the wakeref
243 */
244static inline void
245__intel_wakeref_defer_park(struct intel_wakeref *wf)
246{
247 lockdep_assert_held(&wf->mutex)do { (void)(&wf->mutex); } while(0);
248 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count))((void)0);
249 atomic_set_release(&wf->count, 1)({ typeof(*((&wf->count))) __tmp = (((1))); *(volatile
typeof(*((&wf->count))) *)&(*((&wf->count)
)) = __tmp; __tmp; });
250}
251
252/**
253 * intel_wakeref_wait_for_idle: Wait until the wakeref is idle
254 * @wf: the wakeref
255 *
256 * Wait for the earlier asynchronous release of the wakeref. Note
257 * this will wait for any third party as well, so make sure you only wait
258 * when you have control over the wakeref and trust no one else is acquiring
259 * it.
260 *
261 * Return: 0 on success, error code if killed.
262 */
263int intel_wakeref_wait_for_idle(struct intel_wakeref *wf);
264
265struct intel_wakeref_auto {
266 struct intel_runtime_pm *rpm;
267 struct timeout timer;
268 intel_wakeref_t wakeref;
269 spinlock_t lock;
270 refcount_t count;
271};
272
273/**
274 * intel_wakeref_auto: Delay the runtime-pm autosuspend
275 * @wf: the wakeref
276 * @timeout: relative timeout in jiffies
277 *
278 * The runtime-pm core uses a suspend delay after the last wakeref
279 * is released before triggering runtime suspend of the device. That
280 * delay is configurable via sysfs with little regard to the device
281 * characteristics. Instead, we want to tune the autosuspend based on our
282 * HW knowledge. intel_wakeref_auto() delays the sleep by the supplied
283 * timeout.
284 *
285 * Pass @timeout = 0 to cancel a previous autosuspend by executing the
286 * suspend immediately.
287 */
288void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout);
289
290void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
291 struct intel_runtime_pm *rpm);
292void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf);
293
294#endif /* INTEL_WAKEREF_H */