Bug Summary

File:dev/pci/drm/drm_vblank.c
Warning:line 1914, column 3
Value stored to 'high_prec' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name drm_vblank.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/drm_vblank.c
1/*
2 * drm_irq.c IRQ and vblank support
3 *
4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 * \author Gareth Hughes <gareth@valinux.com>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
25 */
26
27#include <linux/export.h>
28#include <linux/kthread.h>
29#include <linux/moduleparam.h>
30
31#include <drm/drm_crtc.h>
32#include <drm/drm_drv.h>
33#include <drm/drm_framebuffer.h>
34#include <drm/drm_managed.h>
35#include <drm/drm_modeset_helper_vtables.h>
36#include <drm/drm_print.h>
37#include <drm/drm_vblank.h>
38
39#include "drm_internal.h"
40#include "drm_trace.h"
41
42/**
43 * DOC: vblank handling
44 *
45 * From the computer's perspective, every time the monitor displays
46 * a new frame the scanout engine has "scanned out" the display image
47 * from top to bottom, one row of pixels at a time. The current row
48 * of pixels is referred to as the current scanline.
49 *
50 * In addition to the display's visible area, there's usually a couple of
51 * extra scanlines which aren't actually displayed on the screen.
52 * These extra scanlines don't contain image data and are occasionally used
53 * for features like audio and infoframes. The region made up of these
54 * scanlines is referred to as the vertical blanking region, or vblank for
55 * short.
56 *
57 * For historical reference, the vertical blanking period was designed to
58 * give the electron gun (on CRTs) enough time to move back to the top of
59 * the screen to start scanning out the next frame. Similar for horizontal
60 * blanking periods. They were designed to give the electron gun enough
61 * time to move back to the other side of the screen to start scanning the
62 * next scanline.
63 *
64 * ::
65 *
66 *
67 * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
68 * top of | |
69 * display | |
70 * | New frame |
71 * | |
72 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
73 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
74 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the
75 * | | frame as it
76 * | | travels down
77 * | | ("sacn out")
78 * | Old frame |
79 * | |
80 * | |
81 * | |
82 * | | physical
83 * | | bottom of
84 * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
85 * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
86 * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
87 * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
88 * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
89 * new frame
90 *
91 * "Physical top of display" is the reference point for the high-precision/
92 * corrected timestamp.
93 *
94 * On a lot of display hardware, programming needs to take effect during the
95 * vertical blanking period so that settings like gamma, the image buffer
96 * buffer to be scanned out, etc. can safely be changed without showing
97 * any visual artifacts on the screen. In some unforgiving hardware, some of
98 * this programming has to both start and end in the same vblank. To help
99 * with the timing of the hardware programming, an interrupt is usually
100 * available to notify the driver when it can start the updating of registers.
101 * The interrupt is in this context named the vblank interrupt.
102 *
103 * The vblank interrupt may be fired at different points depending on the
104 * hardware. Some hardware implementations will fire the interrupt when the
105 * new frame start, other implementations will fire the interrupt at different
106 * points in time.
107 *
108 * Vertical blanking plays a major role in graphics rendering. To achieve
109 * tear-free display, users must synchronize page flips and/or rendering to
110 * vertical blanking. The DRM API offers ioctls to perform page flips
111 * synchronized to vertical blanking and wait for vertical blanking.
112 *
113 * The DRM core handles most of the vertical blanking management logic, which
114 * involves filtering out spurious interrupts, keeping race-free blanking
115 * counters, coping with counter wrap-around and resets and keeping use counts.
116 * It relies on the driver to generate vertical blanking interrupts and
117 * optionally provide a hardware vertical blanking counter.
118 *
119 * Drivers must initialize the vertical blanking handling core with a call to
120 * drm_vblank_init(). Minimally, a driver needs to implement
121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
123 * support.
124 *
125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
126 * themselves (for instance to handle page flipping operations). The DRM core
127 * maintains a vertical blanking use count to ensure that the interrupts are not
128 * disabled while a user still needs them. To increment the use count, drivers
129 * call drm_crtc_vblank_get() and release the vblank reference again with
130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
131 * guaranteed to be enabled.
132 *
133 * On many hardware disabling the vblank interrupt cannot be done in a race-free
134 * manner, see &drm_driver.vblank_disable_immediate and
135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
136 * vblanks after a timer has expired, which can be configured through the
137 * ``vblankoffdelay`` module parameter.
138 *
139 * Drivers for hardware without support for vertical-blanking interrupts
140 * must not call drm_vblank_init(). For such drivers, atomic helpers will
141 * automatically generate fake vblank events as part of the display update.
142 * This functionality also can be controlled by the driver by enabling and
143 * disabling struct drm_crtc_state.no_vblank.
144 */
145
146/* Retry timestamp calculation up to 3 times to satisfy
147 * drm_timestamp_precision before giving up.
148 */
149#define DRM_TIMESTAMP_MAXRETRIES3 3
150
151/* Threshold in nanoseconds for detection of redundant
152 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
153 */
154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS1000000 1000000
155
156static bool_Bool
157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
158 ktime_t *tvblank, bool_Bool in_vblank_irq);
159
160static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
161
162static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
163
164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
168
169static void store_vblank(struct drm_device *dev, unsigned int pipe,
170 u32 vblank_count_inc,
171 ktime_t t_vblank, u32 last)
172{
173 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
174
175 assert_spin_locked(&dev->vblank_time_lock)do { if (((&dev->vblank_time_lock)->mtx_owner != ({
struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&dev->vblank_time_lock), __func__); } while (0)
;
176
177 vblank->last = last;
178
179 write_seqlock(&vblank->seqlock);
180 vblank->time = t_vblank;
181 atomic64_add(vblank_count_inc, &vblank->count)__sync_fetch_and_add_8(&vblank->count, vblank_count_inc
)
;
182 write_sequnlock(&vblank->seqlock);
183}
184
185static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
186{
187 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
188
189 return vblank->max_vblank_count ?: dev->max_vblank_count;
190}
191
192/*
193 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
194 * if there is no useable hardware frame counter available.
195 */
196static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
197{
198 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0)({ static int __warned; int __ret = !!((drm_max_vblank_count(
dev, pipe) != 0)); if (__ret && !__warned) { printf("%s %s: "
"%s", dev_driver_string(((dev))->dev), "", "drm_WARN_ON_ONCE("
"drm_max_vblank_count(dev, pipe) != 0" ")"); __warned = 1; }
__builtin_expect(!!(__ret), 0); })
;
199 return 0;
200}
201
202static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
203{
204 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
205 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
206
207 if (drm_WARN_ON(dev, !crtc)({ int __ret = !!((!crtc)); if (__ret) printf("%s %s: " "%s",
dev_driver_string(((dev))->dev), "", "drm_WARN_ON(" "!crtc"
")"); __builtin_expect(!!(__ret), 0); })
)
208 return 0;
209
210 if (crtc->funcs->get_vblank_counter)
211 return crtc->funcs->get_vblank_counter(crtc);
212 } else if (dev->driver->get_vblank_counter) {
213 return dev->driver->get_vblank_counter(dev, pipe);
214 }
215
216 return drm_vblank_no_hw_counter(dev, pipe);
217}
218
219/*
220 * Reset the stored timestamp for the current vblank count to correspond
221 * to the last vblank occurred.
222 *
223 * Only to be called from drm_crtc_vblank_on().
224 *
225 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
226 * device vblank fields.
227 */
228static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
229{
230 u32 cur_vblank;
231 bool_Bool rc;
232 ktime_t t_vblank;
233 int count = DRM_TIMESTAMP_MAXRETRIES3;
234
235 spin_lock(&dev->vblank_time_lock)mtx_enter(&dev->vblank_time_lock);
236
237 /*
238 * sample the current counter to avoid random jumps
239 * when drm_vblank_enable() applies the diff
240 */
241 do {
242 cur_vblank = __get_vblank_counter(dev, pipe);
243 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false0);
244 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
245
246 /*
247 * Only reinitialize corresponding vblank timestamp if high-precision query
248 * available and didn't fail. Otherwise reinitialize delayed at next vblank
249 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
250 */
251 if (!rc)
252 t_vblank = 0;
253
254 /*
255 * +1 to make sure user will never see the same
256 * vblank counter value before and after a modeset
257 */
258 store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
259
260 spin_unlock(&dev->vblank_time_lock)mtx_leave(&dev->vblank_time_lock);
261}
262
263/*
264 * Call back into the driver to update the appropriate vblank counter
265 * (specified by @pipe). Deal with wraparound, if it occurred, and
266 * update the last read value so we can deal with wraparound on the next
267 * call if necessary.
268 *
269 * Only necessary when going from off->on, to account for frames we
270 * didn't get an interrupt for.
271 *
272 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
273 * device vblank fields.
274 */
275static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
276 bool_Bool in_vblank_irq)
277{
278 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
279 u32 cur_vblank, diff;
280 bool_Bool rc;
281 ktime_t t_vblank;
282 int count = DRM_TIMESTAMP_MAXRETRIES3;
283 int framedur_ns = vblank->framedur_ns;
284 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
285
286 /*
287 * Interrupts were disabled prior to this call, so deal with counter
288 * wrap if needed.
289 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
290 * here if the register is small or we had vblank interrupts off for
291 * a long time.
292 *
293 * We repeat the hardware vblank counter & timestamp query until
294 * we get consistent results. This to prevent races between gpu
295 * updating its hardware counter while we are retrieving the
296 * corresponding vblank timestamp.
297 */
298 do {
299 cur_vblank = __get_vblank_counter(dev, pipe);
300 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
301 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
302
303 if (max_vblank_count) {
304 /* trust the hw counter when it's around */
305 diff = (cur_vblank - vblank->last) & max_vblank_count;
306 } else if (rc && framedur_ns) {
307 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
308
309 /*
310 * Figure out how many vblanks we've missed based
311 * on the difference in the timestamps and the
312 * frame/field duration.
313 */
314
315 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u: Calculating number of vblanks."
" diff_ns = %lld, framedur_ns = %d)\n", pipe, (long long)diff_ns
, framedur_ns)
316 " diff_ns = %lld, framedur_ns = %d)\n",drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u: Calculating number of vblanks."
" diff_ns = %lld, framedur_ns = %d)\n", pipe, (long long)diff_ns
, framedur_ns)
317 pipe, (long long)diff_ns, framedur_ns)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u: Calculating number of vblanks."
" diff_ns = %lld, framedur_ns = %d)\n", pipe, (long long)diff_ns
, framedur_ns)
;
318
319 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns)(((diff_ns) + ((framedur_ns) / 2)) / (framedur_ns));
320
321 if (diff == 0 && in_vblank_irq)
322 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u: Redundant vblirq ignored\n"
, pipe)
323 pipe)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u: Redundant vblirq ignored\n"
, pipe)
;
324 } else {
325 /* some kind of default for drivers w/o accurate vbl timestamping */
326 diff = in_vblank_irq ? 1 : 0;
327 }
328
329 /*
330 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
331 * interval? If so then vblank irqs keep running and it will likely
332 * happen that the hardware vblank counter is not trustworthy as it
333 * might reset at some point in that interval and vblank timestamps
334 * are not trustworthy either in that interval. Iow. this can result
335 * in a bogus diff >> 1 which must be avoided as it would cause
336 * random large forward jumps of the software vblank counter.
337 */
338 if (diff > 1 && (vblank->inmodeset & 0x2)) {
339 drm_dbg_vbl(dev,drm_dev_dbg((dev)->dev, DRM_UT_VBL, "clamping vblank bump to 1 on crtc %u: diffr=%u"
" due to pre-modeset.\n", pipe, diff)
340 "clamping vblank bump to 1 on crtc %u: diffr=%u"drm_dev_dbg((dev)->dev, DRM_UT_VBL, "clamping vblank bump to 1 on crtc %u: diffr=%u"
" due to pre-modeset.\n", pipe, diff)
341 " due to pre-modeset.\n", pipe, diff)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "clamping vblank bump to 1 on crtc %u: diffr=%u"
" due to pre-modeset.\n", pipe, diff)
;
342 diff = 1;
343 }
344
345 drm_dbg_vbl(dev, "updating vblank count on crtc %u:"drm_dev_dbg((dev)->dev, DRM_UT_VBL, "updating vblank count on crtc %u:"
" current=%llu, diff=%u, hw=%u hw_last=%u\n", pipe, (unsigned
long long)({ typeof(*(&vblank->count)) __tmp = *(volatile
typeof(*(&vblank->count)) *)&(*(&vblank->count
)); membar_datadep_consumer(); __tmp; }), diff, cur_vblank, vblank
->last)
346 " current=%llu, diff=%u, hw=%u hw_last=%u\n",drm_dev_dbg((dev)->dev, DRM_UT_VBL, "updating vblank count on crtc %u:"
" current=%llu, diff=%u, hw=%u hw_last=%u\n", pipe, (unsigned
long long)({ typeof(*(&vblank->count)) __tmp = *(volatile
typeof(*(&vblank->count)) *)&(*(&vblank->count
)); membar_datadep_consumer(); __tmp; }), diff, cur_vblank, vblank
->last)
347 pipe, (unsigned long long)atomic64_read(&vblank->count),drm_dev_dbg((dev)->dev, DRM_UT_VBL, "updating vblank count on crtc %u:"
" current=%llu, diff=%u, hw=%u hw_last=%u\n", pipe, (unsigned
long long)({ typeof(*(&vblank->count)) __tmp = *(volatile
typeof(*(&vblank->count)) *)&(*(&vblank->count
)); membar_datadep_consumer(); __tmp; }), diff, cur_vblank, vblank
->last)
348 diff, cur_vblank, vblank->last)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "updating vblank count on crtc %u:"
" current=%llu, diff=%u, hw=%u hw_last=%u\n", pipe, (unsigned
long long)({ typeof(*(&vblank->count)) __tmp = *(volatile
typeof(*(&vblank->count)) *)&(*(&vblank->count
)); membar_datadep_consumer(); __tmp; }), diff, cur_vblank, vblank
->last)
;
349
350 if (diff == 0) {
351 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last)({ static int __warned; int __ret = !!((cur_vblank != vblank->
last)); if (__ret && !__warned) { printf("%s %s: " "%s"
, dev_driver_string(((dev))->dev), "", "drm_WARN_ON_ONCE("
"cur_vblank != vblank->last" ")"); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
;
352 return;
353 }
354
355 /*
356 * Only reinitialize corresponding vblank timestamp if high-precision query
357 * available and didn't fail, or we were called from the vblank interrupt.
358 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
359 * for now, to mark the vblanktimestamp as invalid.
360 */
361 if (!rc && !in_vblank_irq)
362 t_vblank = 0;
363
364 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
365}
366
367u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
368{
369 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
370 u64 count;
371
372 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
373 return 0;
374
375 count = atomic64_read(&vblank->count)({ typeof(*(&vblank->count)) __tmp = *(volatile typeof
(*(&vblank->count)) *)&(*(&vblank->count));
membar_datadep_consumer(); __tmp; })
;
376
377 /*
378 * This read barrier corresponds to the implicit write barrier of the
379 * write seqlock in store_vblank(). Note that this is the only place
380 * where we need an explicit barrier, since all other access goes
381 * through drm_vblank_count_and_time(), which already has the required
382 * read barrier curtesy of the read seqlock.
383 */
384 smp_rmb()do { __asm volatile("" ::: "memory"); } while (0);
385
386 return count;
387}
388
389/**
390 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
391 * @crtc: which counter to retrieve
392 *
393 * This function is similar to drm_crtc_vblank_count() but this function
394 * interpolates to handle a race with vblank interrupts using the high precision
395 * timestamping support.
396 *
397 * This is mostly useful for hardware that can obtain the scanout position, but
398 * doesn't have a hardware frame counter.
399 */
400u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
401{
402 struct drm_device *dev = crtc->dev;
403 unsigned int pipe = drm_crtc_index(crtc);
404 u64 vblank;
405 unsigned long flags;
406
407 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&({ static int __warned; int __ret = !!(drm_debug_enabled(DRM_UT_VBL
) && !crtc->funcs->get_vblank_timestamp); if (__ret
&& !__warned) { printf("%s %s: " "This function requires support for accurate vblank timestamps."
, dev_driver_string((dev)->dev), ""); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
408 !crtc->funcs->get_vblank_timestamp,({ static int __warned; int __ret = !!(drm_debug_enabled(DRM_UT_VBL
) && !crtc->funcs->get_vblank_timestamp); if (__ret
&& !__warned) { printf("%s %s: " "This function requires support for accurate vblank timestamps."
, dev_driver_string((dev)->dev), ""); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
409 "This function requires support for accurate vblank timestamps.")({ static int __warned; int __ret = !!(drm_debug_enabled(DRM_UT_VBL
) && !crtc->funcs->get_vblank_timestamp); if (__ret
&& !__warned) { printf("%s %s: " "This function requires support for accurate vblank timestamps."
, dev_driver_string((dev)->dev), ""); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
;
410
411 spin_lock_irqsave(&dev->vblank_time_lock, flags)do { flags = 0; mtx_enter(&dev->vblank_time_lock); } while
(0)
;
412
413 drm_update_vblank_count(dev, pipe, false0);
414 vblank = drm_vblank_count(dev, pipe);
415
416 spin_unlock_irqrestore(&dev->vblank_time_lock, flags)do { (void)(flags); mtx_leave(&dev->vblank_time_lock);
} while (0)
;
417
418 return vblank;
419}
420EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
421
422static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
423{
424 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
425 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
426
427 if (drm_WARN_ON(dev, !crtc)({ int __ret = !!((!crtc)); if (__ret) printf("%s %s: " "%s",
dev_driver_string(((dev))->dev), "", "drm_WARN_ON(" "!crtc"
")"); __builtin_expect(!!(__ret), 0); })
)
428 return;
429
430 if (crtc->funcs->disable_vblank)
431 crtc->funcs->disable_vblank(crtc);
432 } else {
433 dev->driver->disable_vblank(dev, pipe);
434 }
435}
436
437/*
438 * Disable vblank irq's on crtc, make sure that last vblank count
439 * of hardware and corresponding consistent software vblank counter
440 * are preserved, even if there are any spurious vblank irq's after
441 * disable.
442 */
443void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
444{
445 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
446 unsigned long irqflags;
447
448 assert_spin_locked(&dev->vbl_lock)do { if (((&dev->vbl_lock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&dev->vbl_lock
), __func__); } while (0)
;
449
450 /* Prevent vblank irq processing while disabling vblank irqs,
451 * so no updates of timestamps or count can happen after we've
452 * disabled. Needed to prevent races in case of delayed irq's.
453 */
454 spin_lock_irqsave(&dev->vblank_time_lock, irqflags)do { irqflags = 0; mtx_enter(&dev->vblank_time_lock); }
while (0)
;
455
456 /*
457 * Update vblank count and disable vblank interrupts only if the
458 * interrupts were enabled. This avoids calling the ->disable_vblank()
459 * operation in atomic context with the hardware potentially runtime
460 * suspended.
461 */
462 if (!vblank->enabled)
463 goto out;
464
465 /*
466 * Update the count and timestamp to maintain the
467 * appearance that the counter has been ticking all along until
468 * this time. This makes the count account for the entire time
469 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
470 */
471 drm_update_vblank_count(dev, pipe, false0);
472 __disable_vblank(dev, pipe);
473 vblank->enabled = false0;
474
475out:
476 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags)do { (void)(irqflags); mtx_leave(&dev->vblank_time_lock
); } while (0)
;
477}
478
479static void vblank_disable_fn(void *arg)
480{
481 struct drm_vblank_crtc *vblank = arg;
482 struct drm_device *dev = vblank->dev;
483 unsigned int pipe = vblank->pipe;
484 unsigned long irqflags;
485
486 spin_lock_irqsave(&dev->vbl_lock, irqflags)do { irqflags = 0; mtx_enter(&dev->vbl_lock); } while (
0)
;
487 if (atomic_read(&vblank->refcount)({ typeof(*(&vblank->refcount)) __tmp = *(volatile typeof
(*(&vblank->refcount)) *)&(*(&vblank->refcount
)); membar_datadep_consumer(); __tmp; })
== 0 && vblank->enabled) {
488 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "disabling vblank on crtc %u\n"
, pipe)
;
489 drm_vblank_disable_and_save(dev, pipe);
490 }
491 spin_unlock_irqrestore(&dev->vbl_lock, irqflags)do { (void)(irqflags); mtx_leave(&dev->vbl_lock); } while
(0)
;
492}
493
494static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
495{
496 struct drm_vblank_crtc *vblank = ptr;
497
498 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&({ int __ret = !!((({ typeof(vblank->enabled) __tmp = *(volatile
typeof(vblank->enabled) *)&(vblank->enabled); membar_datadep_consumer
(); __tmp; }) && drm_core_check_feature(dev, DRIVER_MODESET
))); if (__ret) printf("%s %s: " "%s", dev_driver_string(((dev
))->dev), "", "drm_WARN_ON(" "({ typeof(vblank->enabled) __tmp = *(volatile typeof(vblank->enabled) *)&(vblank->enabled); membar_datadep_consumer(); __tmp; }) && drm_core_check_feature(dev, DRIVER_MODESET)"
")"); __builtin_expect(!!(__ret), 0); })
499 drm_core_check_feature(dev, DRIVER_MODESET))({ int __ret = !!((({ typeof(vblank->enabled) __tmp = *(volatile
typeof(vblank->enabled) *)&(vblank->enabled); membar_datadep_consumer
(); __tmp; }) && drm_core_check_feature(dev, DRIVER_MODESET
))); if (__ret) printf("%s %s: " "%s", dev_driver_string(((dev
))->dev), "", "drm_WARN_ON(" "({ typeof(vblank->enabled) __tmp = *(volatile typeof(vblank->enabled) *)&(vblank->enabled); membar_datadep_consumer(); __tmp; }) && drm_core_check_feature(dev, DRIVER_MODESET)"
")"); __builtin_expect(!!(__ret), 0); })
;
500
501 drm_vblank_destroy_worker(vblank);
502 del_timer_sync(&vblank->disable_timer)timeout_del_barrier((&vblank->disable_timer));
503}
504
505/**
506 * drm_vblank_init - initialize vblank support
507 * @dev: DRM device
508 * @num_crtcs: number of CRTCs supported by @dev
509 *
510 * This function initializes vblank support for @num_crtcs display pipelines.
511 * Cleanup is handled automatically through a cleanup function added with
512 * drmm_add_action_or_reset().
513 *
514 * Returns:
515 * Zero on success or a negative error code on failure.
516 */
517int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
518{
519 int ret;
520 unsigned int i;
521
522 mtx_init(&dev->vbl_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&dev->
vbl_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9) ? 0x9
: ((0x9)))); } while (0)
;
523 mtx_init(&dev->vblank_time_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&dev->
vblank_time_lock), ((((0x9)) > 0x0 && ((0x9)) <
0x9) ? 0x9 : ((0x9)))); } while (0)
;
524
525 dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL(0x0001 | 0x0004));
526 if (!dev->vblank)
527 return -ENOMEM12;
528
529 dev->num_crtcs = num_crtcs;
530
531 for (i = 0; i < num_crtcs; i++) {
532 struct drm_vblank_crtc *vblank = &dev->vblank[i];
533
534 vblank->dev = dev;
535 vblank->pipe = i;
536 init_waitqueue_head(&vblank->queue);
537#ifdef __linux__
538 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
539#else
540 timeout_set(&vblank->disable_timer, vblank_disable_fn, vblank);
541#endif
542 seqlock_init(&vblank->seqlock, IPL_TTY0x9);
543
544 ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
545 vblank);
546 if (ret)
547 return ret;
548
549 ret = drm_vblank_worker_init(vblank);
550 if (ret)
551 return ret;
552 }
553
554 return 0;
555}
556EXPORT_SYMBOL(drm_vblank_init);
557
558/**
559 * drm_dev_has_vblank - test if vblanking has been initialized for
560 * a device
561 * @dev: the device
562 *
563 * Drivers may call this function to test if vblank support is
564 * initialized for a device. For most hardware this means that vblanking
565 * can also be enabled.
566 *
567 * Atomic helpers use this function to initialize
568 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
569 *
570 * Returns:
571 * True if vblanking has been initialized for the given device, false
572 * otherwise.
573 */
574bool_Bool drm_dev_has_vblank(const struct drm_device *dev)
575{
576 return dev->num_crtcs != 0;
577}
578EXPORT_SYMBOL(drm_dev_has_vblank);
579
580/**
581 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
582 * @crtc: which CRTC's vblank waitqueue to retrieve
583 *
584 * This function returns a pointer to the vblank waitqueue for the CRTC.
585 * Drivers can use this to implement vblank waits using wait_event() and related
586 * functions.
587 */
588wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
589{
590 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
591}
592EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
593
594
595/**
596 * drm_calc_timestamping_constants - calculate vblank timestamp constants
597 * @crtc: drm_crtc whose timestamp constants should be updated.
598 * @mode: display mode containing the scanout timings
599 *
600 * Calculate and store various constants which are later needed by vblank and
601 * swap-completion timestamping, e.g, by
602 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
603 * CRTC's true scanout timing, so they take things like panel scaling or
604 * other adjustments into account.
605 */
606void drm_calc_timestamping_constants(struct drm_crtc *crtc,
607 const struct drm_display_mode *mode)
608{
609 struct drm_device *dev = crtc->dev;
610 unsigned int pipe = drm_crtc_index(crtc);
611 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
612 int linedur_ns = 0, framedur_ns = 0;
613 int dotclock = mode->crtc_clock;
614
615 if (!drm_dev_has_vblank(dev))
616 return;
617
618 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
619 return;
620
621 /* Valid dotclock? */
622 if (dotclock > 0) {
623 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
624
625 /*
626 * Convert scanline length in pixels and video
627 * dot clock to line duration and frame duration
628 * in nanoseconds:
629 */
630 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
631 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
632
633 /*
634 * Fields of interlaced scanout modes are only half a frame duration.
635 */
636 if (mode->flags & DRM_MODE_FLAG_INTERLACE(1<<4))
637 framedur_ns /= 2;
638 } else {
639 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "crtc %u: Can't calculate constants, dotclock = 0!\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , crtc->
base.id)
640 crtc->base.id)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "crtc %u: Can't calculate constants, dotclock = 0!\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , crtc->
base.id)
;
641 }
642
643 vblank->linedur_ns = linedur_ns;
644 vblank->framedur_ns = framedur_ns;
645 vblank->hwmode = *mode;
646
647 drm_dbg_core(dev,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n"
, crtc->base.id, mode->crtc_htotal, mode->crtc_vtotal
, mode->crtc_vdisplay)
648 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n"
, crtc->base.id, mode->crtc_htotal, mode->crtc_vtotal
, mode->crtc_vdisplay)
649 crtc->base.id, mode->crtc_htotal,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n"
, crtc->base.id, mode->crtc_htotal, mode->crtc_vtotal
, mode->crtc_vdisplay)
650 mode->crtc_vtotal, mode->crtc_vdisplay)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n"
, crtc->base.id, mode->crtc_htotal, mode->crtc_vtotal
, mode->crtc_vdisplay)
;
651 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: clock %d kHz framedur %d linedur %d\n"
, crtc->base.id, dotclock, framedur_ns, linedur_ns)
652 crtc->base.id, dotclock, framedur_ns, linedur_ns)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: clock %d kHz framedur %d linedur %d\n"
, crtc->base.id, dotclock, framedur_ns, linedur_ns)
;
653}
654EXPORT_SYMBOL(drm_calc_timestamping_constants);
655
656/**
657 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
658 * timestamp helper
659 * @crtc: CRTC whose vblank timestamp to retrieve
660 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
661 * On return contains true maximum error of timestamp
662 * @vblank_time: Pointer to time which should receive the timestamp
663 * @in_vblank_irq:
664 * True when called from drm_crtc_handle_vblank(). Some drivers
665 * need to apply some workarounds for gpu-specific vblank irq quirks
666 * if flag is set.
667 * @get_scanout_position:
668 * Callback function to retrieve the scanout position. See
669 * @struct drm_crtc_helper_funcs.get_scanout_position.
670 *
671 * Implements calculation of exact vblank timestamps from given drm_display_mode
672 * timings and current video scanout position of a CRTC.
673 *
674 * The current implementation only handles standard video modes. For double scan
675 * and interlaced modes the driver is supposed to adjust the hardware mode
676 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
677 * match the scanout position reported.
678 *
679 * Note that atomic drivers must call drm_calc_timestamping_constants() before
680 * enabling a CRTC. The atomic helpers already take care of that in
681 * drm_atomic_helper_calc_timestamping_constants().
682 *
683 * Returns:
684 *
685 * Returns true on success, and false on failure, i.e. when no accurate
686 * timestamp could be acquired.
687 */
688bool_Bool
689drm_crtc_vblank_helper_get_vblank_timestamp_internal(
690 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
691 bool_Bool in_vblank_irq,
692 drm_vblank_get_scanout_position_func get_scanout_position)
693{
694 struct drm_device *dev = crtc->dev;
695 unsigned int pipe = crtc->index;
696 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
697 struct timespec64 ts_etime, ts_vblank_time;
698 ktime_t stime, etime;
699 bool_Bool vbl_status;
700 const struct drm_display_mode *mode;
701 int vpos, hpos, i;
702 int delta_ns, duration_ns;
703
704 if (pipe >= dev->num_crtcs) {
705 drm_err(dev, "Invalid crtc %u\n", pipe)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid crtc %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , pipe)
;
706 return false0;
707 }
708
709 /* Scanout position query not supported? Should not happen. */
710 if (!get_scanout_position) {
711 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Called from CRTC w/o get_scanout_position()!?\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__)
;
712 return false0;
713 }
714
715 if (drm_drv_uses_atomic_modeset(dev))
716 mode = &vblank->hwmode;
717 else
718 mode = &crtc->hwmode;
719
720 /* If mode timing undefined, just return as no-op:
721 * Happens during initial modesetting of a crtc.
722 */
723 if (mode->crtc_clock == 0) {
724 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: Noop due to uninitialized mode.\n"
, pipe)
725 pipe)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: Noop due to uninitialized mode.\n"
, pipe)
;
726 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev))({ static int __warned; int __ret = !!((drm_drv_uses_atomic_modeset
(dev))); if (__ret && !__warned) { printf("%s %s: " "%s"
, dev_driver_string(((dev))->dev), "", "drm_WARN_ON_ONCE("
"drm_drv_uses_atomic_modeset(dev)" ")"); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
;
727 return false0;
728 }
729
730 /* Get current scanout position with system timestamp.
731 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
732 * if single query takes longer than max_error nanoseconds.
733 *
734 * This guarantees a tight bound on maximum error if
735 * code gets preempted or delayed for some reason.
736 */
737 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES3; i++) {
738 /*
739 * Get vertical and horizontal scanout position vpos, hpos,
740 * and bounding timestamps stime, etime, pre/post query.
741 */
742 vbl_status = get_scanout_position(crtc, in_vblank_irq,
743 &vpos, &hpos,
744 &stime, &etime,
745 mode);
746
747 /* Return as no-op if scanout query unsupported or failed. */
748 if (!vbl_status) {
749 drm_dbg_core(dev,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u : scanoutpos query failed.\n"
, pipe)
750 "crtc %u : scanoutpos query failed.\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u : scanoutpos query failed.\n"
, pipe)
751 pipe)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u : scanoutpos query failed.\n"
, pipe)
;
752 return false0;
753 }
754
755 /* Compute uncertainty in timestamp of scanout position query. */
756 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
757
758 /* Accept result with < max_error nsecs timing uncertainty. */
759 if (duration_ns <= *max_error)
760 break;
761 }
762
763 /* Noisy system timing? */
764 if (i == DRM_TIMESTAMP_MAXRETRIES3) {
765 drm_dbg_core(dev,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n"
, pipe, duration_ns / 1000, *max_error / 1000, i)
766 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n"
, pipe, duration_ns / 1000, *max_error / 1000, i)
767 pipe, duration_ns / 1000, *max_error / 1000, i)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n"
, pipe, duration_ns / 1000, *max_error / 1000, i)
;
768 }
769
770 /* Return upper bound of timestamp precision error. */
771 *max_error = duration_ns;
772
773 /* Convert scanout position into elapsed time at raw_time query
774 * since start of scanout at first display scanline. delta_ns
775 * can be negative if start of scanout hasn't happened yet.
776 */
777 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
778 mode->crtc_clock);
779
780 /* Subtract time delta from raw timestamp to get final
781 * vblank_time timestamp for end of vblank.
782 */
783 *vblank_time = ktime_sub_ns(etime, delta_ns);
784
785 if (!drm_debug_enabled(DRM_UT_VBL))
786 return true1;
787
788 ts_etime = ktime_to_timespec64(etime);
789 ts_vblank_time = ktime_to_timespec64(*vblank_time);
790
791 drm_dbg_vbl(dev,drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n"
, pipe, hpos, vpos, (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000
, (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, duration_ns
/ 1000, i)
792 "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n"
, pipe, hpos, vpos, (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000
, (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, duration_ns
/ 1000, i)
793 pipe, hpos, vpos,drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n"
, pipe, hpos, vpos, (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000
, (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, duration_ns
/ 1000, i)
794 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n"
, pipe, hpos, vpos, (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000
, (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, duration_ns
/ 1000, i)
795 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n"
, pipe, hpos, vpos, (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000
, (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, duration_ns
/ 1000, i)
796 duration_ns / 1000, i)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n"
, pipe, hpos, vpos, (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000
, (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, duration_ns
/ 1000, i)
;
797
798 return true1;
799}
800EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
801
802/**
803 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
804 * helper
805 * @crtc: CRTC whose vblank timestamp to retrieve
806 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
807 * On return contains true maximum error of timestamp
808 * @vblank_time: Pointer to time which should receive the timestamp
809 * @in_vblank_irq:
810 * True when called from drm_crtc_handle_vblank(). Some drivers
811 * need to apply some workarounds for gpu-specific vblank irq quirks
812 * if flag is set.
813 *
814 * Implements calculation of exact vblank timestamps from given drm_display_mode
815 * timings and current video scanout position of a CRTC. This can be directly
816 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
817 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
818 *
819 * The current implementation only handles standard video modes. For double scan
820 * and interlaced modes the driver is supposed to adjust the hardware mode
821 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
822 * match the scanout position reported.
823 *
824 * Note that atomic drivers must call drm_calc_timestamping_constants() before
825 * enabling a CRTC. The atomic helpers already take care of that in
826 * drm_atomic_helper_calc_timestamping_constants().
827 *
828 * Returns:
829 *
830 * Returns true on success, and false on failure, i.e. when no accurate
831 * timestamp could be acquired.
832 */
833bool_Bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
834 int *max_error,
835 ktime_t *vblank_time,
836 bool_Bool in_vblank_irq)
837{
838 return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
839 crtc, max_error, vblank_time, in_vblank_irq,
840 crtc->helper_private->get_scanout_position);
841}
842EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
843
844/**
845 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
846 * vblank interval
847 * @dev: DRM device
848 * @pipe: index of CRTC whose vblank timestamp to retrieve
849 * @tvblank: Pointer to target time which should receive the timestamp
850 * @in_vblank_irq:
851 * True when called from drm_crtc_handle_vblank(). Some drivers
852 * need to apply some workarounds for gpu-specific vblank irq quirks
853 * if flag is set.
854 *
855 * Fetches the system timestamp corresponding to the time of the most recent
856 * vblank interval on specified CRTC. May call into kms-driver to
857 * compute the timestamp with a high-precision GPU specific method.
858 *
859 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
860 * call, i.e., it isn't very precisely locked to the true vblank.
861 *
862 * Returns:
863 * True if timestamp is considered to be very precise, false otherwise.
864 */
865static bool_Bool
866drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
867 ktime_t *tvblank, bool_Bool in_vblank_irq)
868{
869 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
870 bool_Bool ret = false0;
871
872 /* Define requested maximum error on timestamps (nanoseconds). */
873 int max_error = (int) drm_timestamp_precision * 1000;
874
875 /* Query driver if possible and precision timestamping enabled. */
876 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
877 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
878
879 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
880 tvblank, in_vblank_irq);
881 }
882
883 /* GPU high precision timestamp query unsupported or failed.
884 * Return current monotonic/gettimeofday timestamp as best estimate.
885 */
886 if (!ret)
887 *tvblank = ktime_get();
888
889 return ret;
890}
891
892/**
893 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
894 * @crtc: which counter to retrieve
895 *
896 * Fetches the "cooked" vblank count value that represents the number of
897 * vblank events since the system was booted, including lost events due to
898 * modesetting activity. Note that this timer isn't correct against a racing
899 * vblank interrupt (since it only reports the software vblank counter), see
900 * drm_crtc_accurate_vblank_count() for such use-cases.
901 *
902 * Note that for a given vblank counter value drm_crtc_handle_vblank()
903 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
904 * provide a barrier: Any writes done before calling
905 * drm_crtc_handle_vblank() will be visible to callers of the later
906 * functions, iff the vblank count is the same or a later one.
907 *
908 * See also &drm_vblank_crtc.count.
909 *
910 * Returns:
911 * The software vblank counter.
912 */
913u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
914{
915 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
916}
917EXPORT_SYMBOL(drm_crtc_vblank_count);
918
919/**
920 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
921 * system timestamp corresponding to that vblank counter value.
922 * @dev: DRM device
923 * @pipe: index of CRTC whose counter to retrieve
924 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
925 *
926 * Fetches the "cooked" vblank count value that represents the number of
927 * vblank events since the system was booted, including lost events due to
928 * modesetting activity. Returns corresponding system timestamp of the time
929 * of the vblank interval that corresponds to the current vblank counter value.
930 *
931 * This is the legacy version of drm_crtc_vblank_count_and_time().
932 */
933static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
934 ktime_t *vblanktime)
935{
936 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
937 u64 vblank_count;
938 unsigned int seq;
939
940 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
) {
941 *vblanktime = 0;
942 return 0;
943 }
944
945 do {
946 seq = read_seqbegin(&vblank->seqlock);
947 vblank_count = atomic64_read(&vblank->count)({ typeof(*(&vblank->count)) __tmp = *(volatile typeof
(*(&vblank->count)) *)&(*(&vblank->count));
membar_datadep_consumer(); __tmp; })
;
948 *vblanktime = vblank->time;
949 } while (read_seqretry(&vblank->seqlock, seq));
950
951 return vblank_count;
952}
953
954/**
955 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
956 * and the system timestamp corresponding to that vblank counter value
957 * @crtc: which counter to retrieve
958 * @vblanktime: Pointer to time to receive the vblank timestamp.
959 *
960 * Fetches the "cooked" vblank count value that represents the number of
961 * vblank events since the system was booted, including lost events due to
962 * modesetting activity. Returns corresponding system timestamp of the time
963 * of the vblank interval that corresponds to the current vblank counter value.
964 *
965 * Note that for a given vblank counter value drm_crtc_handle_vblank()
966 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
967 * provide a barrier: Any writes done before calling
968 * drm_crtc_handle_vblank() will be visible to callers of the later
969 * functions, iff the vblank count is the same or a later one.
970 *
971 * See also &drm_vblank_crtc.count.
972 */
973u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
974 ktime_t *vblanktime)
975{
976 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
977 vblanktime);
978}
979EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
980
981static void send_vblank_event(struct drm_device *dev,
982 struct drm_pending_vblank_event *e,
983 u64 seq, ktime_t now)
984{
985 struct timespec64 tv;
986
987 switch (e->event.base.type) {
988 case DRM_EVENT_VBLANK0x01:
989 case DRM_EVENT_FLIP_COMPLETE0x02:
990 tv = ktime_to_timespec64(now);
991 e->event.vbl.sequence = seq;
992 /*
993 * e->event is a user space structure, with hardcoded unsigned
994 * 32-bit seconds/microseconds. This is safe as we always use
995 * monotonic timestamps since linux-4.15
996 */
997 e->event.vbl.tv_sec = tv.tv_sec;
998 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
999 break;
1000 case DRM_EVENT_CRTC_SEQUENCE0x03:
1001 if (seq)
1002 e->event.seq.sequence = seq;
1003 e->event.seq.time_ns = ktime_to_ns(now);
1004 break;
1005 }
1006 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
1007 drm_send_event_locked(dev, &e->base);
1008}
1009
1010/**
1011 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1012 * @crtc: the source CRTC of the vblank event
1013 * @e: the event to send
1014 *
1015 * A lot of drivers need to generate vblank events for the very next vblank
1016 * interrupt. For example when the page flip interrupt happens when the page
1017 * flip gets armed, but not when it actually executes within the next vblank
1018 * period. This helper function implements exactly the required vblank arming
1019 * behaviour.
1020 *
1021 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1022 * atomic commit must ensure that the next vblank happens at exactly the same
1023 * time as the atomic commit is committed to the hardware. This function itself
1024 * does **not** protect against the next vblank interrupt racing with either this
1025 * function call or the atomic commit operation. A possible sequence could be:
1026 *
1027 * 1. Driver commits new hardware state into vblank-synchronized registers.
1028 * 2. A vblank happens, committing the hardware state. Also the corresponding
1029 * vblank interrupt is fired off and fully processed by the interrupt
1030 * handler.
1031 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1032 * 4. The event is only send out for the next vblank, which is wrong.
1033 *
1034 * An equivalent race can happen when the driver calls
1035 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1036 *
1037 * The only way to make this work safely is to prevent the vblank from firing
1038 * (and the hardware from committing anything else) until the entire atomic
1039 * commit sequence has run to completion. If the hardware does not have such a
1040 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1041 * Instead drivers need to manually send out the event from their interrupt
1042 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1043 * possible race with the hardware committing the atomic update.
1044 *
1045 * Caller must hold a vblank reference for the event @e acquired by a
1046 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1047 */
1048void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1049 struct drm_pending_vblank_event *e)
1050{
1051 struct drm_device *dev = crtc->dev;
1052 unsigned int pipe = drm_crtc_index(crtc);
1053
1054 assert_spin_locked(&dev->event_lock)do { if (((&dev->event_lock)->mtx_owner != ({struct
cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&dev->event_lock), __func__); } while (0)
;
1055
1056 e->pipe = pipe;
1057 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1058 list_add_tail(&e->base.link, &dev->vblank_event_list);
1059}
1060EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1061
1062/**
1063 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1064 * @crtc: the source CRTC of the vblank event
1065 * @e: the event to send
1066 *
1067 * Updates sequence # and timestamp on event for the most recently processed
1068 * vblank, and sends it to userspace. Caller must hold event lock.
1069 *
1070 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1071 * situation, especially to send out events for atomic commit operations.
1072 */
1073void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1074 struct drm_pending_vblank_event *e)
1075{
1076 struct drm_device *dev = crtc->dev;
1077 u64 seq;
1078 unsigned int pipe = drm_crtc_index(crtc);
1079 ktime_t now;
1080
1081 if (drm_dev_has_vblank(dev)) {
1082 seq = drm_vblank_count_and_time(dev, pipe, &now);
1083 } else {
1084 seq = 0;
1085
1086 now = ktime_get();
1087 }
1088 e->pipe = pipe;
1089 send_vblank_event(dev, e, seq, now);
1090}
1091EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1092
1093static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1094{
1095 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1096 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1097
1098 if (drm_WARN_ON(dev, !crtc)({ int __ret = !!((!crtc)); if (__ret) printf("%s %s: " "%s",
dev_driver_string(((dev))->dev), "", "drm_WARN_ON(" "!crtc"
")"); __builtin_expect(!!(__ret), 0); })
)
1099 return 0;
1100
1101 if (crtc->funcs->enable_vblank)
1102 return crtc->funcs->enable_vblank(crtc);
1103 } else if (dev->driver->enable_vblank) {
1104 return dev->driver->enable_vblank(dev, pipe);
1105 }
1106
1107 return -EINVAL22;
1108}
1109
1110static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1111{
1112 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1113 int ret = 0;
1114
1115 assert_spin_locked(&dev->vbl_lock)do { if (((&dev->vbl_lock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&dev->vbl_lock
), __func__); } while (0)
;
1116
1117 spin_lock(&dev->vblank_time_lock)mtx_enter(&dev->vblank_time_lock);
1118
1119 if (!vblank->enabled) {
1120 /*
1121 * Enable vblank irqs under vblank_time_lock protection.
1122 * All vblank count & timestamp updates are held off
1123 * until we are done reinitializing master counter and
1124 * timestamps. Filtercode in drm_handle_vblank() will
1125 * prevent double-accounting of same vblank interval.
1126 */
1127 ret = __enable_vblank(dev, pipe);
1128 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "enabling vblank on crtc %u, ret: %d\n"
, pipe, ret)
1129 pipe, ret)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "enabling vblank on crtc %u, ret: %d\n"
, pipe, ret)
;
1130 if (ret) {
1131 atomic_dec(&vblank->refcount)__sync_fetch_and_sub(&vblank->refcount, 1);
1132 } else {
1133 drm_update_vblank_count(dev, pipe, 0);
1134 /* drm_update_vblank_count() includes a wmb so we just
1135 * need to ensure that the compiler emits the write
1136 * to mark the vblank as enabled after the call
1137 * to drm_update_vblank_count().
1138 */
1139 WRITE_ONCE(vblank->enabled, true)({ typeof(vblank->enabled) __tmp = (1); *(volatile typeof(
vblank->enabled) *)&(vblank->enabled) = __tmp; __tmp
; })
;
1140 }
1141 }
1142
1143 spin_unlock(&dev->vblank_time_lock)mtx_leave(&dev->vblank_time_lock);
1144
1145 return ret;
1146}
1147
1148int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1149{
1150 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1151 unsigned long irqflags;
1152 int ret = 0;
1153
1154 if (!drm_dev_has_vblank(dev))
1155 return -EINVAL22;
1156
1157 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1158 return -EINVAL22;
1159
1160 spin_lock_irqsave(&dev->vbl_lock, irqflags)do { irqflags = 0; mtx_enter(&dev->vbl_lock); } while (
0)
;
1161 /* Going from 0->1 means we have to enable interrupts again */
1162 if (atomic_add_return(1, &vblank->refcount)__sync_add_and_fetch(&vblank->refcount, 1) == 1) {
1163 ret = drm_vblank_enable(dev, pipe);
1164 } else {
1165 if (!vblank->enabled) {
1166 atomic_dec(&vblank->refcount)__sync_fetch_and_sub(&vblank->refcount, 1);
1167 ret = -EINVAL22;
1168 }
1169 }
1170 spin_unlock_irqrestore(&dev->vbl_lock, irqflags)do { (void)(irqflags); mtx_leave(&dev->vbl_lock); } while
(0)
;
1171
1172 return ret;
1173}
1174
1175/**
1176 * drm_crtc_vblank_get - get a reference count on vblank events
1177 * @crtc: which CRTC to own
1178 *
1179 * Acquire a reference count on vblank events to avoid having them disabled
1180 * while in use.
1181 *
1182 * Returns:
1183 * Zero on success or a negative error code on failure.
1184 */
1185int drm_crtc_vblank_get(struct drm_crtc *crtc)
1186{
1187 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1188}
1189EXPORT_SYMBOL(drm_crtc_vblank_get);
1190
1191void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1192{
1193 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1194
1195 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1196 return;
1197
1198 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0)({ int __ret = !!((({ typeof(*(&vblank->refcount)) __tmp
= *(volatile typeof(*(&vblank->refcount)) *)&(*(&
vblank->refcount)); membar_datadep_consumer(); __tmp; }) ==
0)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((dev
))->dev), "", "drm_WARN_ON(" "({ typeof(*(&vblank->refcount)) __tmp = *(volatile typeof(*(&vblank->refcount)) *)&(*(&vblank->refcount)); membar_datadep_consumer(); __tmp; }) == 0"
")"); __builtin_expect(!!(__ret), 0); })
)
1199 return;
1200
1201 /* Last user schedules interrupt disable */
1202 if (atomic_dec_and_test(&vblank->refcount)(__sync_sub_and_fetch((&vblank->refcount), 1) == 0)) {
1203 if (drm_vblank_offdelay == 0)
1204 return;
1205 else if (drm_vblank_offdelay < 0)
1206 vblank_disable_fn(vblank);
1207 else if (!dev->vblank_disable_immediate)
1208 mod_timer(&vblank->disable_timer,
1209 jiffies + ((drm_vblank_offdelay * HZhz)/1000));
1210 }
1211}
1212
1213/**
1214 * drm_crtc_vblank_put - give up ownership of vblank events
1215 * @crtc: which counter to give up
1216 *
1217 * Release ownership of a given vblank counter, turning off interrupts
1218 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1219 */
1220void drm_crtc_vblank_put(struct drm_crtc *crtc)
1221{
1222 drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1223}
1224EXPORT_SYMBOL(drm_crtc_vblank_put);
1225
1226/**
1227 * drm_wait_one_vblank - wait for one vblank
1228 * @dev: DRM device
1229 * @pipe: CRTC index
1230 *
1231 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1232 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1233 * due to lack of driver support or because the crtc is off.
1234 *
1235 * This is the legacy version of drm_crtc_wait_one_vblank().
1236 */
1237void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1238{
1239 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1240 int ret;
1241 u64 last;
1242
1243 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1244 return;
1245
1246#ifdef __OpenBSD__1
1247 /*
1248 * If we're cold, vblank interrupts won't happen even if
1249 * they're turned on by the driver. Just stall long enough
1250 * for a vblank to pass. This assumes a vrefresh of at least
1251 * 25 Hz.
1252 */
1253 if (cold) {
1254 delay(40000)(*delay_func)(40000);
1255 return;
1256 }
1257#endif
1258
1259 ret = drm_vblank_get(dev, pipe);
1260 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",({ int __ret = !!(ret); if (__ret) printf("%s %s: " "vblank not available on crtc %i, ret=%i\n"
, dev_driver_string((dev)->dev), "", pipe, ret); __builtin_expect
(!!(__ret), 0); })
1261 pipe, ret)({ int __ret = !!(ret); if (__ret) printf("%s %s: " "vblank not available on crtc %i, ret=%i\n"
, dev_driver_string((dev)->dev), "", pipe, ret); __builtin_expect
(!!(__ret), 0); })
)
1262 return;
1263
1264 last = drm_vblank_count(dev, pipe);
1265
1266 ret = wait_event_timeout(vblank->queue,({ long __ret = (((uint64_t)(100)) * hz / 1000); if (!(last !=
drm_vblank_count(dev, pipe))) __ret = ({ long ret = (((uint64_t
)(100)) * hz / 1000); do { int __error; unsigned long deadline
; ((!cold) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/drm_vblank.c"
, 1268, "!cold")); mtx_enter(&sch_mtx); deadline = jiffies
+ ret; __error = msleep(&vblank->queue, &sch_mtx,
0, "drmweti", ret); ret = deadline - jiffies; if (__error ==
-1 || __error == 4) { ret = -4; mtx_leave(&sch_mtx); break
; } if (((((uint64_t)(100)) * hz / 1000)) > 0 && (
ret <= 0 || __error == 35)) { mtx_leave(&sch_mtx); ret
= ((last != drm_vblank_count(dev, pipe))) ? 1 : 0; break; } mtx_leave
(&sch_mtx); } while (ret > 0 && !(last != drm_vblank_count
(dev, pipe))); ret; }); __ret; })
1267 last != drm_vblank_count(dev, pipe),({ long __ret = (((uint64_t)(100)) * hz / 1000); if (!(last !=
drm_vblank_count(dev, pipe))) __ret = ({ long ret = (((uint64_t
)(100)) * hz / 1000); do { int __error; unsigned long deadline
; ((!cold) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/drm_vblank.c"
, 1268, "!cold")); mtx_enter(&sch_mtx); deadline = jiffies
+ ret; __error = msleep(&vblank->queue, &sch_mtx,
0, "drmweti", ret); ret = deadline - jiffies; if (__error ==
-1 || __error == 4) { ret = -4; mtx_leave(&sch_mtx); break
; } if (((((uint64_t)(100)) * hz / 1000)) > 0 && (
ret <= 0 || __error == 35)) { mtx_leave(&sch_mtx); ret
= ((last != drm_vblank_count(dev, pipe))) ? 1 : 0; break; } mtx_leave
(&sch_mtx); } while (ret > 0 && !(last != drm_vblank_count
(dev, pipe))); ret; }); __ret; })
1268 msecs_to_jiffies(100))({ long __ret = (((uint64_t)(100)) * hz / 1000); if (!(last !=
drm_vblank_count(dev, pipe))) __ret = ({ long ret = (((uint64_t
)(100)) * hz / 1000); do { int __error; unsigned long deadline
; ((!cold) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/drm_vblank.c"
, 1268, "!cold")); mtx_enter(&sch_mtx); deadline = jiffies
+ ret; __error = msleep(&vblank->queue, &sch_mtx,
0, "drmweti", ret); ret = deadline - jiffies; if (__error ==
-1 || __error == 4) { ret = -4; mtx_leave(&sch_mtx); break
; } if (((((uint64_t)(100)) * hz / 1000)) > 0 && (
ret <= 0 || __error == 35)) { mtx_leave(&sch_mtx); ret
= ((last != drm_vblank_count(dev, pipe))) ? 1 : 0; break; } mtx_leave
(&sch_mtx); } while (ret > 0 && !(last != drm_vblank_count
(dev, pipe))); ret; }); __ret; })
;
1269
1270 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe)({ int __ret = !!(ret == 0); if (__ret) printf("%s %s: " "vblank wait timed out on crtc %i\n"
, dev_driver_string((dev)->dev), "", pipe); __builtin_expect
(!!(__ret), 0); })
;
1271
1272 drm_vblank_put(dev, pipe);
1273}
1274EXPORT_SYMBOL(drm_wait_one_vblank);
1275
1276/**
1277 * drm_crtc_wait_one_vblank - wait for one vblank
1278 * @crtc: DRM crtc
1279 *
1280 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1281 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1282 * due to lack of driver support or because the crtc is off.
1283 */
1284void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1285{
1286 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1287}
1288EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1289
1290/**
1291 * drm_crtc_vblank_off - disable vblank events on a CRTC
1292 * @crtc: CRTC in question
1293 *
1294 * Drivers can use this function to shut down the vblank interrupt handling when
1295 * disabling a crtc. This function ensures that the latest vblank frame count is
1296 * stored so that drm_vblank_on can restore it again.
1297 *
1298 * Drivers must use this function when the hardware vblank counter can get
1299 * reset, e.g. when suspending or disabling the @crtc in general.
1300 */
1301void drm_crtc_vblank_off(struct drm_crtc *crtc)
1302{
1303 struct drm_device *dev = crtc->dev;
1304 unsigned int pipe = drm_crtc_index(crtc);
1305 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1306 struct drm_pending_vblank_event *e, *t;
1307 ktime_t now;
1308 u64 seq;
1309
1310 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1311 return;
1312
1313 /*
1314 * Grab event_lock early to prevent vblank work from being scheduled
1315 * while we're in the middle of shutting down vblank interrupts
1316 */
1317 spin_lock_irq(&dev->event_lock)mtx_enter(&dev->event_lock);
1318
1319 spin_lock(&dev->vbl_lock)mtx_enter(&dev->vbl_lock);
1320 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %d, vblank enabled %d, inmodeset %d\n"
, pipe, vblank->enabled, vblank->inmodeset)
1321 pipe, vblank->enabled, vblank->inmodeset)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %d, vblank enabled %d, inmodeset %d\n"
, pipe, vblank->enabled, vblank->inmodeset)
;
1322
1323 /* Avoid redundant vblank disables without previous
1324 * drm_crtc_vblank_on(). */
1325 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1326 drm_vblank_disable_and_save(dev, pipe);
1327
1328 wake_up(&vblank->queue);
1329
1330 /*
1331 * Prevent subsequent drm_vblank_get() from re-enabling
1332 * the vblank interrupt by bumping the refcount.
1333 */
1334 if (!vblank->inmodeset) {
1335 atomic_inc(&vblank->refcount)__sync_fetch_and_add(&vblank->refcount, 1);
1336 vblank->inmodeset = 1;
1337 }
1338 spin_unlock(&dev->vbl_lock)mtx_leave(&dev->vbl_lock);
1339
1340 /* Send any queued vblank events, lest the natives grow disquiet */
1341 seq = drm_vblank_count_and_time(dev, pipe, &now);
1342
1343 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link)for (e = ({ const __typeof( ((__typeof(*e) *)0)->base.link
) *__mptr = ((&dev->vblank_event_list)->next); (__typeof
(*e) *)( (char *)__mptr - __builtin_offsetof(__typeof(*e), base
.link) );}), t = ({ const __typeof( ((__typeof(*e) *)0)->base
.link ) *__mptr = (e->base.link.next); (__typeof(*e) *)( (
char *)__mptr - __builtin_offsetof(__typeof(*e), base.link) )
;}); &e->base.link != (&dev->vblank_event_list)
; e = t, t = ({ const __typeof( ((__typeof(*t) *)0)->base.
link ) *__mptr = (t->base.link.next); (__typeof(*t) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*t), base.link) );}))
{
1344 if (e->pipe != pipe)
1345 continue;
1346 drm_dbg_core(dev, "Sending premature vblank event on disable: "drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Sending premature vblank event on disable: "
"wanted %llu, current %llu\n", e->sequence, seq)
1347 "wanted %llu, current %llu\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Sending premature vblank event on disable: "
"wanted %llu, current %llu\n", e->sequence, seq)
1348 e->sequence, seq)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Sending premature vblank event on disable: "
"wanted %llu, current %llu\n", e->sequence, seq)
;
1349 list_del(&e->base.link);
1350 drm_vblank_put(dev, pipe);
1351 send_vblank_event(dev, e, seq, now);
1352 }
1353
1354 /* Cancel any leftover pending vblank work */
1355 drm_vblank_cancel_pending_works(vblank);
1356
1357 spin_unlock_irq(&dev->event_lock)mtx_leave(&dev->event_lock);
1358
1359 /* Will be reset by the modeset helpers when re-enabling the crtc by
1360 * calling drm_calc_timestamping_constants(). */
1361 vblank->hwmode.crtc_clock = 0;
1362
1363 /* Wait for any vblank work that's still executing to finish */
1364 drm_vblank_flush_worker(vblank);
1365}
1366EXPORT_SYMBOL(drm_crtc_vblank_off);
1367
1368/**
1369 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1370 * @crtc: CRTC in question
1371 *
1372 * Drivers can use this function to reset the vblank state to off at load time.
1373 * Drivers should use this together with the drm_crtc_vblank_off() and
1374 * drm_crtc_vblank_on() functions. The difference compared to
1375 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1376 * and hence doesn't need to call any driver hooks.
1377 *
1378 * This is useful for recovering driver state e.g. on driver load, or on resume.
1379 */
1380void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1381{
1382 struct drm_device *dev = crtc->dev;
1383 unsigned int pipe = drm_crtc_index(crtc);
1384 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1385
1386 spin_lock_irq(&dev->vbl_lock)mtx_enter(&dev->vbl_lock);
1387 /*
1388 * Prevent subsequent drm_vblank_get() from enabling the vblank
1389 * interrupt by bumping the refcount.
1390 */
1391 if (!vblank->inmodeset) {
1392 atomic_inc(&vblank->refcount)__sync_fetch_and_add(&vblank->refcount, 1);
1393 vblank->inmodeset = 1;
1394 }
1395 spin_unlock_irq(&dev->vbl_lock)mtx_leave(&dev->vbl_lock);
1396
1397 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list))({ int __ret = !!((!list_empty(&dev->vblank_event_list
))); if (__ret) printf("%s %s: " "%s", dev_driver_string(((dev
))->dev), "", "drm_WARN_ON(" "!list_empty(&dev->vblank_event_list)"
")"); __builtin_expect(!!(__ret), 0); })
;
1398 drm_WARN_ON(dev, !list_empty(&vblank->pending_work))({ int __ret = !!((!list_empty(&vblank->pending_work))
); if (__ret) printf("%s %s: " "%s", dev_driver_string(((dev)
)->dev), "", "drm_WARN_ON(" "!list_empty(&vblank->pending_work)"
")"); __builtin_expect(!!(__ret), 0); })
;
1399}
1400EXPORT_SYMBOL(drm_crtc_vblank_reset);
1401
1402/**
1403 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1404 * @crtc: CRTC in question
1405 * @max_vblank_count: max hardware vblank counter value
1406 *
1407 * Update the maximum hardware vblank counter value for @crtc
1408 * at runtime. Useful for hardware where the operation of the
1409 * hardware vblank counter depends on the currently active
1410 * display configuration.
1411 *
1412 * For example, if the hardware vblank counter does not work
1413 * when a specific connector is active the maximum can be set
1414 * to zero. And when that specific connector isn't active the
1415 * maximum can again be set to the appropriate non-zero value.
1416 *
1417 * If used, must be called before drm_vblank_on().
1418 */
1419void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1420 u32 max_vblank_count)
1421{
1422 struct drm_device *dev = crtc->dev;
1423 unsigned int pipe = drm_crtc_index(crtc);
1424 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1425
1426 drm_WARN_ON(dev, dev->max_vblank_count)({ int __ret = !!((dev->max_vblank_count)); if (__ret) printf
("%s %s: " "%s", dev_driver_string(((dev))->dev), "", "drm_WARN_ON("
"dev->max_vblank_count" ")"); __builtin_expect(!!(__ret),
0); })
;
1427 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset))({ int __ret = !!((!({ typeof(vblank->inmodeset) __tmp = *
(volatile typeof(vblank->inmodeset) *)&(vblank->inmodeset
); membar_datadep_consumer(); __tmp; }))); if (__ret) printf(
"%s %s: " "%s", dev_driver_string(((dev))->dev), "", "drm_WARN_ON("
"!({ typeof(vblank->inmodeset) __tmp = *(volatile typeof(vblank->inmodeset) *)&(vblank->inmodeset); membar_datadep_consumer(); __tmp; })"
")"); __builtin_expect(!!(__ret), 0); })
;
1428
1429 vblank->max_vblank_count = max_vblank_count;
1430}
1431EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1432
1433/**
1434 * drm_crtc_vblank_on - enable vblank events on a CRTC
1435 * @crtc: CRTC in question
1436 *
1437 * This functions restores the vblank interrupt state captured with
1438 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1439 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1440 * unbalanced and so can also be unconditionally called in driver load code to
1441 * reflect the current hardware state of the crtc.
1442 */
1443void drm_crtc_vblank_on(struct drm_crtc *crtc)
1444{
1445 struct drm_device *dev = crtc->dev;
1446 unsigned int pipe = drm_crtc_index(crtc);
1447 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1448
1449 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1450 return;
1451
1452 spin_lock_irq(&dev->vbl_lock)mtx_enter(&dev->vbl_lock);
1453 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %d, vblank enabled %d, inmodeset %d\n"
, pipe, vblank->enabled, vblank->inmodeset)
1454 pipe, vblank->enabled, vblank->inmodeset)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "crtc %d, vblank enabled %d, inmodeset %d\n"
, pipe, vblank->enabled, vblank->inmodeset)
;
1455
1456 /* Drop our private "prevent drm_vblank_get" refcount */
1457 if (vblank->inmodeset) {
1458 atomic_dec(&vblank->refcount)__sync_fetch_and_sub(&vblank->refcount, 1);
1459 vblank->inmodeset = 0;
1460 }
1461
1462 drm_reset_vblank_timestamp(dev, pipe);
1463
1464 /*
1465 * re-enable interrupts if there are users left, or the
1466 * user wishes vblank interrupts to be enabled all the time.
1467 */
1468 if (atomic_read(&vblank->refcount)({ typeof(*(&vblank->refcount)) __tmp = *(volatile typeof
(*(&vblank->refcount)) *)&(*(&vblank->refcount
)); membar_datadep_consumer(); __tmp; })
!= 0 || drm_vblank_offdelay == 0)
1469 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe))({ int __ret = !!((drm_vblank_enable(dev, pipe))); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "drm_vblank_enable(dev, pipe)" ")"); __builtin_expect
(!!(__ret), 0); })
;
1470 spin_unlock_irq(&dev->vbl_lock)mtx_leave(&dev->vbl_lock);
1471}
1472EXPORT_SYMBOL(drm_crtc_vblank_on);
1473
1474/**
1475 * drm_vblank_restore - estimate missed vblanks and update vblank count.
1476 * @dev: DRM device
1477 * @pipe: CRTC index
1478 *
1479 * Power manamement features can cause frame counter resets between vblank
1480 * disable and enable. Drivers can use this function in their
1481 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1482 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1483 * vblank counter.
1484 *
1485 * This function is the legacy version of drm_crtc_vblank_restore().
1486 */
1487void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1488{
1489 ktime_t t_vblank;
1490 struct drm_vblank_crtc *vblank;
1491 int framedur_ns;
1492 u64 diff_ns;
1493 u32 cur_vblank, diff = 1;
1494 int count = DRM_TIMESTAMP_MAXRETRIES3;
1495
1496 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1497 return;
1498
1499 assert_spin_locked(&dev->vbl_lock)do { if (((&dev->vbl_lock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&dev->vbl_lock
), __func__); } while (0)
;
1500 assert_spin_locked(&dev->vblank_time_lock)do { if (((&dev->vblank_time_lock)->mtx_owner != ({
struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&dev->vblank_time_lock), __func__); } while (0)
;
1501
1502 vblank = &dev->vblank[pipe];
1503 drm_WARN_ONCE(dev,({ static int __warned; int __ret = !!(drm_debug_enabled(DRM_UT_VBL
) && !vblank->framedur_ns); if (__ret && !
__warned) { printf("%s %s: " "Cannot compute missed vblanks without frame duration\n"
, dev_driver_string((dev)->dev), ""); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
1504 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,({ static int __warned; int __ret = !!(drm_debug_enabled(DRM_UT_VBL
) && !vblank->framedur_ns); if (__ret && !
__warned) { printf("%s %s: " "Cannot compute missed vblanks without frame duration\n"
, dev_driver_string((dev)->dev), ""); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
1505 "Cannot compute missed vblanks without frame duration\n")({ static int __warned; int __ret = !!(drm_debug_enabled(DRM_UT_VBL
) && !vblank->framedur_ns); if (__ret && !
__warned) { printf("%s %s: " "Cannot compute missed vblanks without frame duration\n"
, dev_driver_string((dev)->dev), ""); __warned = 1; } __builtin_expect
(!!(__ret), 0); })
;
1506 framedur_ns = vblank->framedur_ns;
1507
1508 do {
1509 cur_vblank = __get_vblank_counter(dev, pipe);
1510 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false0);
1511 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1512
1513 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1514 if (framedur_ns)
1515 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns)(((diff_ns) + ((framedur_ns) / 2)) / (framedur_ns));
1516
1517
1518 drm_dbg_vbl(dev,drm_dev_dbg((dev)->dev, DRM_UT_VBL, "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n"
, diff, diff_ns, framedur_ns, cur_vblank - vblank->last)
1519 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",drm_dev_dbg((dev)->dev, DRM_UT_VBL, "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n"
, diff, diff_ns, framedur_ns, cur_vblank - vblank->last)
1520 diff, diff_ns, framedur_ns, cur_vblank - vblank->last)drm_dev_dbg((dev)->dev, DRM_UT_VBL, "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n"
, diff, diff_ns, framedur_ns, cur_vblank - vblank->last)
;
1521 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
1522}
1523EXPORT_SYMBOL(drm_vblank_restore);
1524
1525/**
1526 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1527 * @crtc: CRTC in question
1528 *
1529 * Power manamement features can cause frame counter resets between vblank
1530 * disable and enable. Drivers can use this function in their
1531 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1532 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1533 * vblank counter.
1534 */
1535void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1536{
1537 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1538}
1539EXPORT_SYMBOL(drm_crtc_vblank_restore);
1540
1541static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1542 unsigned int pipe)
1543{
1544 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1545
1546 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1547 if (!drm_dev_has_vblank(dev))
1548 return;
1549
1550 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1551 return;
1552
1553 /*
1554 * To avoid all the problems that might happen if interrupts
1555 * were enabled/disabled around or between these calls, we just
1556 * have the kernel take a reference on the CRTC (just once though
1557 * to avoid corrupting the count if multiple, mismatch calls occur),
1558 * so that interrupts remain enabled in the interim.
1559 */
1560 if (!vblank->inmodeset) {
1561 vblank->inmodeset = 0x1;
1562 if (drm_vblank_get(dev, pipe) == 0)
1563 vblank->inmodeset |= 0x2;
1564 }
1565}
1566
1567static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1568 unsigned int pipe)
1569{
1570 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1571
1572 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1573 if (!drm_dev_has_vblank(dev))
1574 return;
1575
1576 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1577 return;
1578
1579 if (vblank->inmodeset) {
1580 spin_lock_irq(&dev->vbl_lock)mtx_enter(&dev->vbl_lock);
1581 drm_reset_vblank_timestamp(dev, pipe);
1582 spin_unlock_irq(&dev->vbl_lock)mtx_leave(&dev->vbl_lock);
1583
1584 if (vblank->inmodeset & 0x2)
1585 drm_vblank_put(dev, pipe);
1586
1587 vblank->inmodeset = 0;
1588 }
1589}
1590
1591int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1592 struct drm_file *file_priv)
1593{
1594 struct drm_modeset_ctl *modeset = data;
1595 unsigned int pipe;
1596
1597 /* If drm_vblank_init() hasn't been called yet, just no-op */
1598 if (!drm_dev_has_vblank(dev))
1599 return 0;
1600
1601 /* KMS drivers handle this internally */
1602 if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1603 return 0;
1604
1605 pipe = modeset->crtc;
1606 if (pipe >= dev->num_crtcs)
1607 return -EINVAL22;
1608
1609 switch (modeset->cmd) {
1610 case _DRM_PRE_MODESET1:
1611 drm_legacy_vblank_pre_modeset(dev, pipe);
1612 break;
1613 case _DRM_POST_MODESET2:
1614 drm_legacy_vblank_post_modeset(dev, pipe);
1615 break;
1616 default:
1617 return -EINVAL22;
1618 }
1619
1620 return 0;
1621}
1622
1623static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1624 u64 req_seq,
1625 union drm_wait_vblank *vblwait,
1626 struct drm_file *file_priv)
1627{
1628 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1629 struct drm_pending_vblank_event *e;
1630 ktime_t now;
1631 u64 seq;
1632 int ret;
1633
1634 e = kzalloc(sizeof(*e), GFP_KERNEL(0x0001 | 0x0004));
1635 if (e == NULL((void *)0)) {
1636 ret = -ENOMEM12;
1637 goto err_put;
1638 }
1639
1640 e->pipe = pipe;
1641 e->event.base.type = DRM_EVENT_VBLANK0x01;
1642 e->event.base.length = sizeof(e->event.vbl);
1643 e->event.vbl.user_data = vblwait->request.signal;
1644 e->event.vbl.crtc_id = 0;
1645 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1646 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1647
1648 if (crtc)
1649 e->event.vbl.crtc_id = crtc->base.id;
1650 }
1651
1652 spin_lock_irq(&dev->event_lock)mtx_enter(&dev->event_lock);
1653
1654 /*
1655 * drm_crtc_vblank_off() might have been called after we called
1656 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1657 * vblank disable, so no need for further locking. The reference from
1658 * drm_vblank_get() protects against vblank disable from another source.
1659 */
1660 if (!READ_ONCE(vblank->enabled)({ typeof(vblank->enabled) __tmp = *(volatile typeof(vblank
->enabled) *)&(vblank->enabled); membar_datadep_consumer
(); __tmp; })
) {
1661 ret = -EINVAL22;
1662 goto err_unlock;
1663 }
1664
1665 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1666 &e->event.base);
1667
1668 if (ret)
1669 goto err_unlock;
1670
1671 seq = drm_vblank_count_and_time(dev, pipe, &now);
1672
1673 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "event on vblank count %llu, current %llu, crtc %u\n"
, req_seq, seq, pipe)
1674 req_seq, seq, pipe)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "event on vblank count %llu, current %llu, crtc %u\n"
, req_seq, seq, pipe)
;
1675
1676 trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1677
1678 e->sequence = req_seq;
1679 if (drm_vblank_passed(seq, req_seq)) {
1680 drm_vblank_put(dev, pipe);
1681 send_vblank_event(dev, e, seq, now);
1682 vblwait->reply.sequence = seq;
1683 } else {
1684 /* drm_handle_vblank_events will call drm_vblank_put */
1685 list_add_tail(&e->base.link, &dev->vblank_event_list);
1686 vblwait->reply.sequence = req_seq;
1687 }
1688
1689 spin_unlock_irq(&dev->event_lock)mtx_leave(&dev->event_lock);
1690
1691 return 0;
1692
1693err_unlock:
1694 spin_unlock_irq(&dev->event_lock)mtx_leave(&dev->event_lock);
1695 kfree(e);
1696err_put:
1697 drm_vblank_put(dev, pipe);
1698 return ret;
1699}
1700
1701static bool_Bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1702{
1703 if (vblwait->request.sequence)
1704 return false0;
1705
1706 return _DRM_VBLANK_RELATIVE ==
1707 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK(_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE) |
1708 _DRM_VBLANK_EVENT |
1709 _DRM_VBLANK_NEXTONMISS));
1710}
1711
1712/*
1713 * Widen a 32-bit param to 64-bits.
1714 *
1715 * \param narrow 32-bit value (missing upper 32 bits)
1716 * \param near 64-bit value that should be 'close' to near
1717 *
1718 * This function returns a 64-bit value using the lower 32-bits from
1719 * 'narrow' and constructing the upper 32-bits so that the result is
1720 * as close as possible to 'near'.
1721 */
1722
1723static u64 widen_32_to_64(u32 narrow, u64 near)
1724{
1725 return near + (s32) (narrow - near);
1726}
1727
1728static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1729 struct drm_wait_vblank_reply *reply)
1730{
1731 ktime_t now;
1732 struct timespec64 ts;
1733
1734 /*
1735 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1736 * to store the seconds. This is safe as we always use monotonic
1737 * timestamps since linux-4.15.
1738 */
1739 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1740 ts = ktime_to_timespec64(now);
1741 reply->tval_sec = (u32)ts.tv_sec;
1742 reply->tval_usec = ts.tv_nsec / 1000;
1743}
1744
1745int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1746 struct drm_file *file_priv)
1747{
1748 struct drm_crtc *crtc;
1749 struct drm_vblank_crtc *vblank;
1750 union drm_wait_vblank *vblwait = data;
1751 int ret;
1752 u64 req_seq, seq;
1753 unsigned int pipe_index;
1754 unsigned int flags, pipe, high_pipe;
1755
1756 if (!dev->irq_enabled)
1757 return -EOPNOTSUPP45;
1758
1759 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1760 return -EINVAL22;
1761
1762 if (vblwait->request.type &
1763 ~(_DRM_VBLANK_TYPES_MASK(_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE) | _DRM_VBLANK_FLAGS_MASK(_DRM_VBLANK_EVENT | _DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY
| _DRM_VBLANK_NEXTONMISS)
|
1764 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1765 drm_dbg_core(dev,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Unsupported type value 0x%x, supported mask 0x%x\n"
, vblwait->request.type, ((_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE
) | (_DRM_VBLANK_EVENT | _DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY
| _DRM_VBLANK_NEXTONMISS) | _DRM_VBLANK_HIGH_CRTC_MASK))
1766 "Unsupported type value 0x%x, supported mask 0x%x\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Unsupported type value 0x%x, supported mask 0x%x\n"
, vblwait->request.type, ((_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE
) | (_DRM_VBLANK_EVENT | _DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY
| _DRM_VBLANK_NEXTONMISS) | _DRM_VBLANK_HIGH_CRTC_MASK))
1767 vblwait->request.type,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Unsupported type value 0x%x, supported mask 0x%x\n"
, vblwait->request.type, ((_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE
) | (_DRM_VBLANK_EVENT | _DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY
| _DRM_VBLANK_NEXTONMISS) | _DRM_VBLANK_HIGH_CRTC_MASK))
1768 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Unsupported type value 0x%x, supported mask 0x%x\n"
, vblwait->request.type, ((_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE
) | (_DRM_VBLANK_EVENT | _DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY
| _DRM_VBLANK_NEXTONMISS) | _DRM_VBLANK_HIGH_CRTC_MASK))
1769 _DRM_VBLANK_HIGH_CRTC_MASK))drm_dev_dbg((dev)->dev, DRM_UT_CORE, "Unsupported type value 0x%x, supported mask 0x%x\n"
, vblwait->request.type, ((_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE
) | (_DRM_VBLANK_EVENT | _DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY
| _DRM_VBLANK_NEXTONMISS) | _DRM_VBLANK_HIGH_CRTC_MASK))
;
1770 return -EINVAL22;
1771 }
1772
1773 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK(_DRM_VBLANK_EVENT | _DRM_VBLANK_SIGNAL | _DRM_VBLANK_SECONDARY
| _DRM_VBLANK_NEXTONMISS)
;
1774 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1775 if (high_pipe)
1776 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT1;
1777 else
1778 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1779
1780 /* Convert lease-relative crtc index into global crtc index */
1781 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1782 pipe = 0;
1783 drm_for_each_crtc(crtc, dev)for (crtc = ({ const __typeof( ((__typeof(*crtc) *)0)->head
) *__mptr = ((&(dev)->mode_config.crtc_list)->next
); (__typeof(*crtc) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*crtc), head) );}); &crtc->head != (&(dev)->mode_config
.crtc_list); crtc = ({ const __typeof( ((__typeof(*crtc) *)0)
->head ) *__mptr = (crtc->head.next); (__typeof(*crtc) *
)( (char *)__mptr - __builtin_offsetof(__typeof(*crtc), head)
);}))
{
1784 if (drm_lease_held(file_priv, crtc->base.id)(1)) {
1785 if (pipe_index == 0)
1786 break;
1787 pipe_index--;
1788 }
1789 pipe++;
1790 }
1791 } else {
1792 pipe = pipe_index;
1793 }
1794
1795 if (pipe >= dev->num_crtcs)
1796 return -EINVAL22;
1797
1798 vblank = &dev->vblank[pipe];
1799
1800 /* If the counter is currently enabled and accurate, short-circuit
1801 * queries to return the cached timestamp of the last vblank.
1802 */
1803 if (dev->vblank_disable_immediate &&
1804 drm_wait_vblank_is_query(vblwait) &&
1805 READ_ONCE(vblank->enabled)({ typeof(vblank->enabled) __tmp = *(volatile typeof(vblank
->enabled) *)&(vblank->enabled); membar_datadep_consumer
(); __tmp; })
) {
1806 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1807 return 0;
1808 }
1809
1810 ret = drm_vblank_get(dev, pipe);
1811 if (ret) {
1812 drm_dbg_core(dev,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
1813 "crtc %d failed to acquire vblank counter, %d\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
1814 pipe, ret)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
;
1815 return ret;
1816 }
1817 seq = drm_vblank_count(dev, pipe);
1818
1819 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK(_DRM_VBLANK_ABSOLUTE | _DRM_VBLANK_RELATIVE)) {
1820 case _DRM_VBLANK_RELATIVE:
1821 req_seq = seq + vblwait->request.sequence;
1822 vblwait->request.sequence = req_seq;
1823 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1824 break;
1825 case _DRM_VBLANK_ABSOLUTE:
1826 req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1827 break;
1828 default:
1829 ret = -EINVAL22;
1830 goto done;
1831 }
1832
1833 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1834 drm_vblank_passed(seq, req_seq)) {
1835 req_seq = seq + 1;
1836 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1837 vblwait->request.sequence = req_seq;
1838 }
1839
1840 if (flags & _DRM_VBLANK_EVENT) {
1841 /* must hold on to the vblank ref until the event fires
1842 * drm_vblank_put will be called asynchronously
1843 */
1844 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1845 }
1846
1847 if (req_seq != seq) {
1848 int wait;
1849
1850 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "waiting on vblank count %llu, crtc %u\n"
, req_seq, pipe)
1851 req_seq, pipe)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "waiting on vblank count %llu, crtc %u\n"
, req_seq, pipe)
;
1852 wait = wait_event_interruptible_timeout(vblank->queue,({ long __ret = (((uint64_t)(3000)) * hz / 1000); if (!(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; }))) __ret
= ({ long ret = (((uint64_t)(3000)) * hz / 1000); do { int __error
; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/pci/drm/drm_vblank.c", 1855, "!cold")); mtx_enter
(&sch_mtx); deadline = jiffies + ret; __error = msleep(&
vblank->queue, &sch_mtx, 0x100, "drmweti", ret); ret =
deadline - jiffies; if (__error == -1 || __error == 4) { ret
= -4; mtx_leave(&sch_mtx); break; } if (((((uint64_t)(3000
)) * hz / 1000)) > 0 && (ret <= 0 || __error ==
35)) { mtx_leave(&sch_mtx); ret = ((drm_vblank_passed(drm_vblank_count
(dev, pipe), req_seq) || !({ typeof(vblank->enabled) __tmp
= *(volatile typeof(vblank->enabled) *)&(vblank->enabled
); membar_datadep_consumer(); __tmp; }))) ? 1 : 0; break; } mtx_leave
(&sch_mtx); } while (ret > 0 && !(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; })));
ret; }); __ret; })
1853 drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||({ long __ret = (((uint64_t)(3000)) * hz / 1000); if (!(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; }))) __ret
= ({ long ret = (((uint64_t)(3000)) * hz / 1000); do { int __error
; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/pci/drm/drm_vblank.c", 1855, "!cold")); mtx_enter
(&sch_mtx); deadline = jiffies + ret; __error = msleep(&
vblank->queue, &sch_mtx, 0x100, "drmweti", ret); ret =
deadline - jiffies; if (__error == -1 || __error == 4) { ret
= -4; mtx_leave(&sch_mtx); break; } if (((((uint64_t)(3000
)) * hz / 1000)) > 0 && (ret <= 0 || __error ==
35)) { mtx_leave(&sch_mtx); ret = ((drm_vblank_passed(drm_vblank_count
(dev, pipe), req_seq) || !({ typeof(vblank->enabled) __tmp
= *(volatile typeof(vblank->enabled) *)&(vblank->enabled
); membar_datadep_consumer(); __tmp; }))) ? 1 : 0; break; } mtx_leave
(&sch_mtx); } while (ret > 0 && !(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; })));
ret; }); __ret; })
1854 !READ_ONCE(vblank->enabled),({ long __ret = (((uint64_t)(3000)) * hz / 1000); if (!(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; }))) __ret
= ({ long ret = (((uint64_t)(3000)) * hz / 1000); do { int __error
; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/pci/drm/drm_vblank.c", 1855, "!cold")); mtx_enter
(&sch_mtx); deadline = jiffies + ret; __error = msleep(&
vblank->queue, &sch_mtx, 0x100, "drmweti", ret); ret =
deadline - jiffies; if (__error == -1 || __error == 4) { ret
= -4; mtx_leave(&sch_mtx); break; } if (((((uint64_t)(3000
)) * hz / 1000)) > 0 && (ret <= 0 || __error ==
35)) { mtx_leave(&sch_mtx); ret = ((drm_vblank_passed(drm_vblank_count
(dev, pipe), req_seq) || !({ typeof(vblank->enabled) __tmp
= *(volatile typeof(vblank->enabled) *)&(vblank->enabled
); membar_datadep_consumer(); __tmp; }))) ? 1 : 0; break; } mtx_leave
(&sch_mtx); } while (ret > 0 && !(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; })));
ret; }); __ret; })
1855 msecs_to_jiffies(3000))({ long __ret = (((uint64_t)(3000)) * hz / 1000); if (!(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; }))) __ret
= ({ long ret = (((uint64_t)(3000)) * hz / 1000); do { int __error
; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/dev/pci/drm/drm_vblank.c", 1855, "!cold")); mtx_enter
(&sch_mtx); deadline = jiffies + ret; __error = msleep(&
vblank->queue, &sch_mtx, 0x100, "drmweti", ret); ret =
deadline - jiffies; if (__error == -1 || __error == 4) { ret
= -4; mtx_leave(&sch_mtx); break; } if (((((uint64_t)(3000
)) * hz / 1000)) > 0 && (ret <= 0 || __error ==
35)) { mtx_leave(&sch_mtx); ret = ((drm_vblank_passed(drm_vblank_count
(dev, pipe), req_seq) || !({ typeof(vblank->enabled) __tmp
= *(volatile typeof(vblank->enabled) *)&(vblank->enabled
); membar_datadep_consumer(); __tmp; }))) ? 1 : 0; break; } mtx_leave
(&sch_mtx); } while (ret > 0 && !(drm_vblank_passed
(drm_vblank_count(dev, pipe), req_seq) || !({ typeof(vblank->
enabled) __tmp = *(volatile typeof(vblank->enabled) *)&
(vblank->enabled); membar_datadep_consumer(); __tmp; })));
ret; }); __ret; })
;
1856
1857 switch (wait) {
1858 case 0:
1859 /* timeout */
1860 ret = -EBUSY16;
1861 break;
1862 case -ERESTARTSYS4:
1863 /* interrupted by signal */
1864 ret = -EINTR4;
1865 break;
1866 default:
1867 ret = 0;
1868 break;
1869 }
1870 }
1871
1872 if (ret != -EINTR4) {
1873 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1874
1875 drm_dbg_core(dev, "crtc %d returning %u to client\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d returning %u to client\n"
, pipe, vblwait->reply.sequence)
1876 pipe, vblwait->reply.sequence)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d returning %u to client\n"
, pipe, vblwait->reply.sequence)
;
1877 } else {
1878 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d vblank wait interrupted by signal\n"
, pipe)
1879 pipe)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d vblank wait interrupted by signal\n"
, pipe)
;
1880 }
1881
1882done:
1883 drm_vblank_put(dev, pipe);
1884 return ret;
1885}
1886
1887static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1888{
1889 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1890 bool_Bool high_prec = false0;
1891 struct drm_pending_vblank_event *e, *t;
1892 ktime_t now;
1893 u64 seq;
1894
1895 assert_spin_locked(&dev->event_lock)do { if (((&dev->event_lock)->mtx_owner != ({struct
cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&dev->event_lock), __func__); } while (0)
;
1896
1897 seq = drm_vblank_count_and_time(dev, pipe, &now);
1898
1899 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link)for (e = ({ const __typeof( ((__typeof(*e) *)0)->base.link
) *__mptr = ((&dev->vblank_event_list)->next); (__typeof
(*e) *)( (char *)__mptr - __builtin_offsetof(__typeof(*e), base
.link) );}), t = ({ const __typeof( ((__typeof(*e) *)0)->base
.link ) *__mptr = (e->base.link.next); (__typeof(*e) *)( (
char *)__mptr - __builtin_offsetof(__typeof(*e), base.link) )
;}); &e->base.link != (&dev->vblank_event_list)
; e = t, t = ({ const __typeof( ((__typeof(*t) *)0)->base.
link ) *__mptr = (t->base.link.next); (__typeof(*t) *)( (char
*)__mptr - __builtin_offsetof(__typeof(*t), base.link) );}))
{
1900 if (e->pipe != pipe)
1901 continue;
1902 if (!drm_vblank_passed(seq, e->sequence))
1903 continue;
1904
1905 drm_dbg_core(dev, "vblank event on %llu, current %llu\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "vblank event on %llu, current %llu\n"
, e->sequence, seq)
1906 e->sequence, seq)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "vblank event on %llu, current %llu\n"
, e->sequence, seq)
;
1907
1908 list_del(&e->base.link);
1909 drm_vblank_put(dev, pipe);
1910 send_vblank_event(dev, e, seq, now);
1911 }
1912
1913 if (crtc && crtc->funcs->get_vblank_timestamp)
1914 high_prec = true1;
Value stored to 'high_prec' is never read
1915
1916 trace_drm_vblank_event(pipe, seq, now, high_prec);
1917}
1918
1919/**
1920 * drm_handle_vblank - handle a vblank event
1921 * @dev: DRM device
1922 * @pipe: index of CRTC where this event occurred
1923 *
1924 * Drivers should call this routine in their vblank interrupt handlers to
1925 * update the vblank counter and send any signals that may be pending.
1926 *
1927 * This is the legacy version of drm_crtc_handle_vblank().
1928 */
1929bool_Bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1930{
1931 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1932 unsigned long irqflags;
1933 bool_Bool disable_irq;
1934
1935 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev))({ static int __warned; int __ret = !!((!drm_dev_has_vblank(dev
))); if (__ret && !__warned) { printf("%s %s: " "%s",
dev_driver_string(((dev))->dev), "", "drm_WARN_ON_ONCE(" "!drm_dev_has_vblank(dev)"
")"); __warned = 1; } __builtin_expect(!!(__ret), 0); })
)
1936 return false0;
1937
1938 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)({ int __ret = !!((pipe >= dev->num_crtcs)); if (__ret)
printf("%s %s: " "%s", dev_driver_string(((dev))->dev), ""
, "drm_WARN_ON(" "pipe >= dev->num_crtcs" ")"); __builtin_expect
(!!(__ret), 0); })
)
1939 return false0;
1940
1941 spin_lock_irqsave(&dev->event_lock, irqflags)do { irqflags = 0; mtx_enter(&dev->event_lock); } while
(0)
;
1942
1943 /* Need timestamp lock to prevent concurrent execution with
1944 * vblank enable/disable, as this would cause inconsistent
1945 * or corrupted timestamps and vblank counts.
1946 */
1947 spin_lock(&dev->vblank_time_lock)mtx_enter(&dev->vblank_time_lock);
1948
1949 /* Vblank irq handling disabled. Nothing to do. */
1950 if (!vblank->enabled) {
1951 spin_unlock(&dev->vblank_time_lock)mtx_leave(&dev->vblank_time_lock);
1952 spin_unlock_irqrestore(&dev->event_lock, irqflags)do { (void)(irqflags); mtx_leave(&dev->event_lock); } while
(0)
;
1953 return false0;
1954 }
1955
1956 drm_update_vblank_count(dev, pipe, true1);
1957
1958 spin_unlock(&dev->vblank_time_lock)mtx_leave(&dev->vblank_time_lock);
1959
1960 wake_up(&vblank->queue);
1961
1962 /* With instant-off, we defer disabling the interrupt until after
1963 * we finish processing the following vblank after all events have
1964 * been signaled. The disable has to be last (after
1965 * drm_handle_vblank_events) so that the timestamp is always accurate.
1966 */
1967 disable_irq = (dev->vblank_disable_immediate &&
1968 drm_vblank_offdelay > 0 &&
1969 !atomic_read(&vblank->refcount)({ typeof(*(&vblank->refcount)) __tmp = *(volatile typeof
(*(&vblank->refcount)) *)&(*(&vblank->refcount
)); membar_datadep_consumer(); __tmp; })
);
1970
1971 drm_handle_vblank_events(dev, pipe);
1972 drm_handle_vblank_works(vblank);
1973
1974 spin_unlock_irqrestore(&dev->event_lock, irqflags)do { (void)(irqflags); mtx_leave(&dev->event_lock); } while
(0)
;
1975
1976 if (disable_irq)
1977 vblank_disable_fn(vblank);
1978
1979 return true1;
1980}
1981EXPORT_SYMBOL(drm_handle_vblank);
1982
1983/**
1984 * drm_crtc_handle_vblank - handle a vblank event
1985 * @crtc: where this event occurred
1986 *
1987 * Drivers should call this routine in their vblank interrupt handlers to
1988 * update the vblank counter and send any signals that may be pending.
1989 *
1990 * This is the native KMS version of drm_handle_vblank().
1991 *
1992 * Note that for a given vblank counter value drm_crtc_handle_vblank()
1993 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
1994 * provide a barrier: Any writes done before calling
1995 * drm_crtc_handle_vblank() will be visible to callers of the later
1996 * functions, iff the vblank count is the same or a later one.
1997 *
1998 * See also &drm_vblank_crtc.count.
1999 *
2000 * Returns:
2001 * True if the event was successfully handled, false on failure.
2002 */
2003bool_Bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
2004{
2005 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
2006}
2007EXPORT_SYMBOL(drm_crtc_handle_vblank);
2008
2009/*
2010 * Get crtc VBLANK count.
2011 *
2012 * \param dev DRM device
2013 * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
2014 * \param file_priv drm file private for the user's open file descriptor
2015 */
2016
2017int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
2018 struct drm_file *file_priv)
2019{
2020 struct drm_crtc *crtc;
2021 struct drm_vblank_crtc *vblank;
2022 int pipe;
2023 struct drm_crtc_get_sequence *get_seq = data;
2024 ktime_t now;
2025 bool_Bool vblank_enabled;
2026 int ret;
2027
2028 if (!drm_core_check_feature(dev, DRIVER_MODESET))
2029 return -EOPNOTSUPP45;
2030
2031 if (!dev->irq_enabled)
2032 return -EOPNOTSUPP45;
2033
2034 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
2035 if (!crtc)
2036 return -ENOENT2;
2037
2038 pipe = drm_crtc_index(crtc);
2039
2040 vblank = &dev->vblank[pipe];
2041 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled)({ typeof(vblank->enabled) __tmp = *(volatile typeof(vblank
->enabled) *)&(vblank->enabled); membar_datadep_consumer
(); __tmp; })
;
2042
2043 if (!vblank_enabled) {
2044 ret = drm_crtc_vblank_get(crtc);
2045 if (ret) {
2046 drm_dbg_core(dev,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
2047 "crtc %d failed to acquire vblank counter, %d\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
2048 pipe, ret)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
;
2049 return ret;
2050 }
2051 }
2052 drm_modeset_lock(&crtc->mutex, NULL((void *)0));
2053 if (crtc->state)
2054 get_seq->active = crtc->state->enable;
2055 else
2056 get_seq->active = crtc->enabled;
2057 drm_modeset_unlock(&crtc->mutex);
2058 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
2059 get_seq->sequence_ns = ktime_to_ns(now);
2060 if (!vblank_enabled)
2061 drm_crtc_vblank_put(crtc);
2062 return 0;
2063}
2064
2065/*
2066 * Queue a event for VBLANK sequence
2067 *
2068 * \param dev DRM device
2069 * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
2070 * \param file_priv drm file private for the user's open file descriptor
2071 */
2072
2073int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2074 struct drm_file *file_priv)
2075{
2076 struct drm_crtc *crtc;
2077 struct drm_vblank_crtc *vblank;
2078 int pipe;
2079 struct drm_crtc_queue_sequence *queue_seq = data;
2080 ktime_t now;
2081 struct drm_pending_vblank_event *e;
2082 u32 flags;
2083 u64 seq;
2084 u64 req_seq;
2085 int ret;
2086
2087 if (!drm_core_check_feature(dev, DRIVER_MODESET))
2088 return -EOPNOTSUPP45;
2089
2090 if (!dev->irq_enabled)
2091 return -EOPNOTSUPP45;
2092
2093 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
2094 if (!crtc)
2095 return -ENOENT2;
2096
2097 flags = queue_seq->flags;
2098 /* Check valid flag bits */
2099 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE0x00000001|
2100 DRM_CRTC_SEQUENCE_NEXT_ON_MISS0x00000002))
2101 return -EINVAL22;
2102
2103 pipe = drm_crtc_index(crtc);
2104
2105 vblank = &dev->vblank[pipe];
2106
2107 e = kzalloc(sizeof(*e), GFP_KERNEL(0x0001 | 0x0004));
2108 if (e == NULL((void *)0))
2109 return -ENOMEM12;
2110
2111 ret = drm_crtc_vblank_get(crtc);
2112 if (ret) {
2113 drm_dbg_core(dev,drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
2114 "crtc %d failed to acquire vblank counter, %d\n",drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
2115 pipe, ret)drm_dev_dbg((dev)->dev, DRM_UT_CORE, "crtc %d failed to acquire vblank counter, %d\n"
, pipe, ret)
;
2116 goto err_free;
2117 }
2118
2119 seq = drm_vblank_count_and_time(dev, pipe, &now);
2120 req_seq = queue_seq->sequence;
2121
2122 if (flags & DRM_CRTC_SEQUENCE_RELATIVE0x00000001)
2123 req_seq += seq;
2124
2125 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS0x00000002) && drm_vblank_passed(seq, req_seq))
2126 req_seq = seq + 1;
2127
2128 e->pipe = pipe;
2129 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE0x03;
2130 e->event.base.length = sizeof(e->event.seq);
2131 e->event.seq.user_data = queue_seq->user_data;
2132
2133 spin_lock_irq(&dev->event_lock)mtx_enter(&dev->event_lock);
2134
2135 /*
2136 * drm_crtc_vblank_off() might have been called after we called
2137 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2138 * vblank disable, so no need for further locking. The reference from
2139 * drm_crtc_vblank_get() protects against vblank disable from another source.
2140 */
2141 if (!READ_ONCE(vblank->enabled)({ typeof(vblank->enabled) __tmp = *(volatile typeof(vblank
->enabled) *)&(vblank->enabled); membar_datadep_consumer
(); __tmp; })
) {
2142 ret = -EINVAL22;
2143 goto err_unlock;
2144 }
2145
2146 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
2147 &e->event.base);
2148
2149 if (ret)
2150 goto err_unlock;
2151
2152 e->sequence = req_seq;
2153
2154 if (drm_vblank_passed(seq, req_seq)) {
2155 drm_crtc_vblank_put(crtc);
2156 send_vblank_event(dev, e, seq, now);
2157 queue_seq->sequence = seq;
2158 } else {
2159 /* drm_handle_vblank_events will call drm_vblank_put */
2160 list_add_tail(&e->base.link, &dev->vblank_event_list);
2161 queue_seq->sequence = req_seq;
2162 }
2163
2164 spin_unlock_irq(&dev->event_lock)mtx_leave(&dev->event_lock);
2165 return 0;
2166
2167err_unlock:
2168 spin_unlock_irq(&dev->event_lock)mtx_leave(&dev->event_lock);
2169 drm_crtc_vblank_put(crtc);
2170err_free:
2171 kfree(e);
2172 return ret;
2173}
2174