Line data Source code
1 : /*
2 : * drm_irq.c IRQ and vblank support
3 : *
4 : * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 : * \author Gareth Hughes <gareth@valinux.com>
6 : *
7 : * Permission is hereby granted, free of charge, to any person obtaining a
8 : * copy of this software and associated documentation files (the "Software"),
9 : * to deal in the Software without restriction, including without limitation
10 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 : * and/or sell copies of the Software, and to permit persons to whom the
12 : * Software is furnished to do so, subject to the following conditions:
13 : *
14 : * The above copyright notice and this permission notice (including the next
15 : * paragraph) shall be included in all copies or substantial portions of the
16 : * Software.
17 : *
18 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 : * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 : * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 : * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 : * OTHER DEALINGS IN THE SOFTWARE.
25 : */
26 :
27 : #include <linux/export.h>
28 : #include <linux/kthread.h>
29 : #include <linux/moduleparam.h>
30 :
31 : #include <drm/drm_crtc.h>
32 : #include <drm/drm_drv.h>
33 : #include <drm/drm_framebuffer.h>
34 : #include <drm/drm_managed.h>
35 : #include <drm/drm_modeset_helper_vtables.h>
36 : #include <drm/drm_print.h>
37 : #include <drm/drm_vblank.h>
38 :
39 : #include "drm_internal.h"
40 : #include "drm_trace.h"
41 :
42 : /**
43 : * DOC: vblank handling
44 : *
45 : * From the computer's perspective, every time the monitor displays
46 : * a new frame the scanout engine has "scanned out" the display image
47 : * from top to bottom, one row of pixels at a time. The current row
48 : * of pixels is referred to as the current scanline.
49 : *
50 : * In addition to the display's visible area, there's usually a couple of
51 : * extra scanlines which aren't actually displayed on the screen.
52 : * These extra scanlines don't contain image data and are occasionally used
53 : * for features like audio and infoframes. The region made up of these
54 : * scanlines is referred to as the vertical blanking region, or vblank for
55 : * short.
56 : *
57 : * For historical reference, the vertical blanking period was designed to
58 : * give the electron gun (on CRTs) enough time to move back to the top of
59 : * the screen to start scanning out the next frame. Similar for horizontal
60 : * blanking periods. They were designed to give the electron gun enough
61 : * time to move back to the other side of the screen to start scanning the
62 : * next scanline.
63 : *
64 : * ::
65 : *
66 : *
67 : * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
68 : * top of | |
69 : * display | |
70 : * | New frame |
71 : * | |
72 : * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
73 : * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
74 : * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the
75 : * | | frame as it
76 : * | | travels down
77 : * | | ("scan out")
78 : * | Old frame |
79 : * | |
80 : * | |
81 : * | |
82 : * | | physical
83 : * | | bottom of
84 : * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
85 : * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
86 : * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
87 : * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
88 : * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
89 : * new frame
90 : *
91 : * "Physical top of display" is the reference point for the high-precision/
92 : * corrected timestamp.
93 : *
94 : * On a lot of display hardware, programming needs to take effect during the
95 : * vertical blanking period so that settings like gamma, the image buffer
96 : * buffer to be scanned out, etc. can safely be changed without showing
97 : * any visual artifacts on the screen. In some unforgiving hardware, some of
98 : * this programming has to both start and end in the same vblank. To help
99 : * with the timing of the hardware programming, an interrupt is usually
100 : * available to notify the driver when it can start the updating of registers.
101 : * The interrupt is in this context named the vblank interrupt.
102 : *
103 : * The vblank interrupt may be fired at different points depending on the
104 : * hardware. Some hardware implementations will fire the interrupt when the
105 : * new frame start, other implementations will fire the interrupt at different
106 : * points in time.
107 : *
108 : * Vertical blanking plays a major role in graphics rendering. To achieve
109 : * tear-free display, users must synchronize page flips and/or rendering to
110 : * vertical blanking. The DRM API offers ioctls to perform page flips
111 : * synchronized to vertical blanking and wait for vertical blanking.
112 : *
113 : * The DRM core handles most of the vertical blanking management logic, which
114 : * involves filtering out spurious interrupts, keeping race-free blanking
115 : * counters, coping with counter wrap-around and resets and keeping use counts.
116 : * It relies on the driver to generate vertical blanking interrupts and
117 : * optionally provide a hardware vertical blanking counter.
118 : *
119 : * Drivers must initialize the vertical blanking handling core with a call to
120 : * drm_vblank_init(). Minimally, a driver needs to implement
121 : * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
122 : * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
123 : * support.
124 : *
125 : * Vertical blanking interrupts can be enabled by the DRM core or by drivers
126 : * themselves (for instance to handle page flipping operations). The DRM core
127 : * maintains a vertical blanking use count to ensure that the interrupts are not
128 : * disabled while a user still needs them. To increment the use count, drivers
129 : * call drm_crtc_vblank_get() and release the vblank reference again with
130 : * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
131 : * guaranteed to be enabled.
132 : *
133 : * On many hardware disabling the vblank interrupt cannot be done in a race-free
134 : * manner, see &drm_driver.vblank_disable_immediate and
135 : * &drm_driver.max_vblank_count. In that case the vblank core only disables the
136 : * vblanks after a timer has expired, which can be configured through the
137 : * ``vblankoffdelay`` module parameter.
138 : *
139 : * Drivers for hardware without support for vertical-blanking interrupts
140 : * must not call drm_vblank_init(). For such drivers, atomic helpers will
141 : * automatically generate fake vblank events as part of the display update.
142 : * This functionality also can be controlled by the driver by enabling and
143 : * disabling struct drm_crtc_state.no_vblank.
144 : */
145 :
146 : /* Retry timestamp calculation up to 3 times to satisfy
147 : * drm_timestamp_precision before giving up.
148 : */
149 : #define DRM_TIMESTAMP_MAXRETRIES 3
150 :
151 : /* Threshold in nanoseconds for detection of redundant
152 : * vblank irq in drm_handle_vblank(). 1 msec should be ok.
153 : */
154 : #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
155 :
156 : static bool
157 : drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
158 : ktime_t *tvblank, bool in_vblank_irq);
159 :
160 : static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
161 :
162 : static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
163 :
164 : module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
165 : module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
166 : MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
167 : MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
168 :
169 : static void store_vblank(struct drm_device *dev, unsigned int pipe,
170 : u32 vblank_count_inc,
171 : ktime_t t_vblank, u32 last)
172 : {
173 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
174 :
175 : assert_spin_locked(&dev->vblank_time_lock);
176 :
177 0 : vblank->last = last;
178 :
179 0 : write_seqlock(&vblank->seqlock);
180 0 : vblank->time = t_vblank;
181 0 : atomic64_add(vblank_count_inc, &vblank->count);
182 0 : write_sequnlock(&vblank->seqlock);
183 : }
184 :
185 : static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
186 : {
187 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
188 :
189 0 : return vblank->max_vblank_count ?: dev->max_vblank_count;
190 : }
191 :
192 : /*
193 : * "No hw counter" fallback implementation of .get_vblank_counter() hook,
194 : * if there is no usable hardware frame counter available.
195 : */
196 0 : static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
197 : {
198 0 : drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
199 0 : return 0;
200 : }
201 :
202 0 : static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
203 : {
204 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
205 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
206 :
207 0 : if (drm_WARN_ON(dev, !crtc))
208 : return 0;
209 :
210 0 : if (crtc->funcs->get_vblank_counter)
211 0 : return crtc->funcs->get_vblank_counter(crtc);
212 : }
213 : #ifdef CONFIG_DRM_LEGACY
214 : else if (dev->driver->get_vblank_counter) {
215 : return dev->driver->get_vblank_counter(dev, pipe);
216 : }
217 : #endif
218 :
219 0 : return drm_vblank_no_hw_counter(dev, pipe);
220 : }
221 :
222 : /*
223 : * Reset the stored timestamp for the current vblank count to correspond
224 : * to the last vblank occurred.
225 : *
226 : * Only to be called from drm_crtc_vblank_on().
227 : *
228 : * Note: caller must hold &drm_device.vbl_lock since this reads & writes
229 : * device vblank fields.
230 : */
231 0 : static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
232 : {
233 : u32 cur_vblank;
234 : bool rc;
235 : ktime_t t_vblank;
236 0 : int count = DRM_TIMESTAMP_MAXRETRIES;
237 :
238 0 : spin_lock(&dev->vblank_time_lock);
239 :
240 : /*
241 : * sample the current counter to avoid random jumps
242 : * when drm_vblank_enable() applies the diff
243 : */
244 : do {
245 0 : cur_vblank = __get_vblank_counter(dev, pipe);
246 0 : rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
247 0 : } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
248 :
249 : /*
250 : * Only reinitialize corresponding vblank timestamp if high-precision query
251 : * available and didn't fail. Otherwise reinitialize delayed at next vblank
252 : * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
253 : */
254 0 : if (!rc)
255 0 : t_vblank = 0;
256 :
257 : /*
258 : * +1 to make sure user will never see the same
259 : * vblank counter value before and after a modeset
260 : */
261 0 : store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
262 :
263 0 : spin_unlock(&dev->vblank_time_lock);
264 0 : }
265 :
266 : /*
267 : * Call back into the driver to update the appropriate vblank counter
268 : * (specified by @pipe). Deal with wraparound, if it occurred, and
269 : * update the last read value so we can deal with wraparound on the next
270 : * call if necessary.
271 : *
272 : * Only necessary when going from off->on, to account for frames we
273 : * didn't get an interrupt for.
274 : *
275 : * Note: caller must hold &drm_device.vbl_lock since this reads & writes
276 : * device vblank fields.
277 : */
278 0 : static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
279 : bool in_vblank_irq)
280 : {
281 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
282 : u32 cur_vblank, diff;
283 : bool rc;
284 : ktime_t t_vblank;
285 0 : int count = DRM_TIMESTAMP_MAXRETRIES;
286 0 : int framedur_ns = vblank->framedur_ns;
287 0 : u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
288 :
289 : /*
290 : * Interrupts were disabled prior to this call, so deal with counter
291 : * wrap if needed.
292 : * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
293 : * here if the register is small or we had vblank interrupts off for
294 : * a long time.
295 : *
296 : * We repeat the hardware vblank counter & timestamp query until
297 : * we get consistent results. This to prevent races between gpu
298 : * updating its hardware counter while we are retrieving the
299 : * corresponding vblank timestamp.
300 : */
301 : do {
302 0 : cur_vblank = __get_vblank_counter(dev, pipe);
303 0 : rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
304 0 : } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
305 :
306 0 : if (max_vblank_count) {
307 : /* trust the hw counter when it's around */
308 0 : diff = (cur_vblank - vblank->last) & max_vblank_count;
309 0 : } else if (rc && framedur_ns) {
310 0 : u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
311 :
312 : /*
313 : * Figure out how many vblanks we've missed based
314 : * on the difference in the timestamps and the
315 : * frame/field duration.
316 : */
317 :
318 0 : drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
319 : " diff_ns = %lld, framedur_ns = %d)\n",
320 : pipe, (long long)diff_ns, framedur_ns);
321 :
322 0 : diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
323 :
324 0 : if (diff == 0 && in_vblank_irq)
325 0 : drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
326 : pipe);
327 : } else {
328 : /* some kind of default for drivers w/o accurate vbl timestamping */
329 0 : diff = in_vblank_irq ? 1 : 0;
330 : }
331 :
332 : /*
333 : * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
334 : * interval? If so then vblank irqs keep running and it will likely
335 : * happen that the hardware vblank counter is not trustworthy as it
336 : * might reset at some point in that interval and vblank timestamps
337 : * are not trustworthy either in that interval. Iow. this can result
338 : * in a bogus diff >> 1 which must be avoided as it would cause
339 : * random large forward jumps of the software vblank counter.
340 : */
341 0 : if (diff > 1 && (vblank->inmodeset & 0x2)) {
342 0 : drm_dbg_vbl(dev,
343 : "clamping vblank bump to 1 on crtc %u: diffr=%u"
344 : " due to pre-modeset.\n", pipe, diff);
345 0 : diff = 1;
346 : }
347 :
348 0 : drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
349 : " current=%llu, diff=%u, hw=%u hw_last=%u\n",
350 : pipe, (unsigned long long)atomic64_read(&vblank->count),
351 : diff, cur_vblank, vblank->last);
352 :
353 0 : if (diff == 0) {
354 0 : drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
355 0 : return;
356 : }
357 :
358 : /*
359 : * Only reinitialize corresponding vblank timestamp if high-precision query
360 : * available and didn't fail, or we were called from the vblank interrupt.
361 : * Otherwise reinitialize delayed at next vblank interrupt and assign 0
362 : * for now, to mark the vblanktimestamp as invalid.
363 : */
364 0 : if (!rc && !in_vblank_irq)
365 0 : t_vblank = 0;
366 :
367 0 : store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
368 : }
369 :
370 0 : u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
371 : {
372 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
373 : u64 count;
374 :
375 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
376 : return 0;
377 :
378 0 : count = atomic64_read(&vblank->count);
379 :
380 : /*
381 : * This read barrier corresponds to the implicit write barrier of the
382 : * write seqlock in store_vblank(). Note that this is the only place
383 : * where we need an explicit barrier, since all other access goes
384 : * through drm_vblank_count_and_time(), which already has the required
385 : * read barrier curtesy of the read seqlock.
386 : */
387 0 : smp_rmb();
388 :
389 0 : return count;
390 : }
391 :
392 : /**
393 : * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
394 : * @crtc: which counter to retrieve
395 : *
396 : * This function is similar to drm_crtc_vblank_count() but this function
397 : * interpolates to handle a race with vblank interrupts using the high precision
398 : * timestamping support.
399 : *
400 : * This is mostly useful for hardware that can obtain the scanout position, but
401 : * doesn't have a hardware frame counter.
402 : */
403 0 : u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
404 : {
405 0 : struct drm_device *dev = crtc->dev;
406 0 : unsigned int pipe = drm_crtc_index(crtc);
407 : u64 vblank;
408 : unsigned long flags;
409 :
410 0 : drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
411 : !crtc->funcs->get_vblank_timestamp,
412 : "This function requires support for accurate vblank timestamps.");
413 :
414 0 : spin_lock_irqsave(&dev->vblank_time_lock, flags);
415 :
416 0 : drm_update_vblank_count(dev, pipe, false);
417 0 : vblank = drm_vblank_count(dev, pipe);
418 :
419 0 : spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
420 :
421 0 : return vblank;
422 : }
423 : EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
424 :
425 0 : static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
426 : {
427 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
428 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
429 :
430 0 : if (drm_WARN_ON(dev, !crtc))
431 : return;
432 :
433 0 : if (crtc->funcs->disable_vblank)
434 0 : crtc->funcs->disable_vblank(crtc);
435 : }
436 : #ifdef CONFIG_DRM_LEGACY
437 : else {
438 : dev->driver->disable_vblank(dev, pipe);
439 : }
440 : #endif
441 : }
442 :
443 : /*
444 : * Disable vblank irq's on crtc, make sure that last vblank count
445 : * of hardware and corresponding consistent software vblank counter
446 : * are preserved, even if there are any spurious vblank irq's after
447 : * disable.
448 : */
449 0 : void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
450 : {
451 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
452 : unsigned long irqflags;
453 :
454 : assert_spin_locked(&dev->vbl_lock);
455 :
456 : /* Prevent vblank irq processing while disabling vblank irqs,
457 : * so no updates of timestamps or count can happen after we've
458 : * disabled. Needed to prevent races in case of delayed irq's.
459 : */
460 0 : spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
461 :
462 : /*
463 : * Update vblank count and disable vblank interrupts only if the
464 : * interrupts were enabled. This avoids calling the ->disable_vblank()
465 : * operation in atomic context with the hardware potentially runtime
466 : * suspended.
467 : */
468 0 : if (!vblank->enabled)
469 : goto out;
470 :
471 : /*
472 : * Update the count and timestamp to maintain the
473 : * appearance that the counter has been ticking all along until
474 : * this time. This makes the count account for the entire time
475 : * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
476 : */
477 0 : drm_update_vblank_count(dev, pipe, false);
478 0 : __disable_vblank(dev, pipe);
479 0 : vblank->enabled = false;
480 :
481 : out:
482 0 : spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
483 0 : }
484 :
485 0 : static void vblank_disable_fn(struct timer_list *t)
486 : {
487 0 : struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
488 0 : struct drm_device *dev = vblank->dev;
489 0 : unsigned int pipe = vblank->pipe;
490 : unsigned long irqflags;
491 :
492 0 : spin_lock_irqsave(&dev->vbl_lock, irqflags);
493 0 : if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
494 0 : drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
495 0 : drm_vblank_disable_and_save(dev, pipe);
496 : }
497 0 : spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
498 0 : }
499 :
500 0 : static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
501 : {
502 0 : struct drm_vblank_crtc *vblank = ptr;
503 :
504 0 : drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
505 : drm_core_check_feature(dev, DRIVER_MODESET));
506 :
507 0 : drm_vblank_destroy_worker(vblank);
508 0 : del_timer_sync(&vblank->disable_timer);
509 0 : }
510 :
511 : /**
512 : * drm_vblank_init - initialize vblank support
513 : * @dev: DRM device
514 : * @num_crtcs: number of CRTCs supported by @dev
515 : *
516 : * This function initializes vblank support for @num_crtcs display pipelines.
517 : * Cleanup is handled automatically through a cleanup function added with
518 : * drmm_add_action_or_reset().
519 : *
520 : * Returns:
521 : * Zero on success or a negative error code on failure.
522 : */
523 0 : int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
524 : {
525 : int ret;
526 : unsigned int i;
527 :
528 0 : spin_lock_init(&dev->vbl_lock);
529 0 : spin_lock_init(&dev->vblank_time_lock);
530 :
531 0 : dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
532 0 : if (!dev->vblank)
533 : return -ENOMEM;
534 :
535 0 : dev->num_crtcs = num_crtcs;
536 :
537 0 : for (i = 0; i < num_crtcs; i++) {
538 0 : struct drm_vblank_crtc *vblank = &dev->vblank[i];
539 :
540 0 : vblank->dev = dev;
541 0 : vblank->pipe = i;
542 0 : init_waitqueue_head(&vblank->queue);
543 0 : timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
544 0 : seqlock_init(&vblank->seqlock);
545 :
546 0 : ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
547 : vblank);
548 0 : if (ret)
549 : return ret;
550 :
551 0 : ret = drm_vblank_worker_init(vblank);
552 0 : if (ret)
553 : return ret;
554 : }
555 :
556 : return 0;
557 : }
558 : EXPORT_SYMBOL(drm_vblank_init);
559 :
560 : /**
561 : * drm_dev_has_vblank - test if vblanking has been initialized for
562 : * a device
563 : * @dev: the device
564 : *
565 : * Drivers may call this function to test if vblank support is
566 : * initialized for a device. For most hardware this means that vblanking
567 : * can also be enabled.
568 : *
569 : * Atomic helpers use this function to initialize
570 : * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
571 : *
572 : * Returns:
573 : * True if vblanking has been initialized for the given device, false
574 : * otherwise.
575 : */
576 0 : bool drm_dev_has_vblank(const struct drm_device *dev)
577 : {
578 0 : return dev->num_crtcs != 0;
579 : }
580 : EXPORT_SYMBOL(drm_dev_has_vblank);
581 :
582 : /**
583 : * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
584 : * @crtc: which CRTC's vblank waitqueue to retrieve
585 : *
586 : * This function returns a pointer to the vblank waitqueue for the CRTC.
587 : * Drivers can use this to implement vblank waits using wait_event() and related
588 : * functions.
589 : */
590 0 : wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
591 : {
592 0 : return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
593 : }
594 : EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
595 :
596 :
597 : /**
598 : * drm_calc_timestamping_constants - calculate vblank timestamp constants
599 : * @crtc: drm_crtc whose timestamp constants should be updated.
600 : * @mode: display mode containing the scanout timings
601 : *
602 : * Calculate and store various constants which are later needed by vblank and
603 : * swap-completion timestamping, e.g, by
604 : * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
605 : * CRTC's true scanout timing, so they take things like panel scaling or
606 : * other adjustments into account.
607 : */
608 0 : void drm_calc_timestamping_constants(struct drm_crtc *crtc,
609 : const struct drm_display_mode *mode)
610 : {
611 0 : struct drm_device *dev = crtc->dev;
612 0 : unsigned int pipe = drm_crtc_index(crtc);
613 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
614 0 : int linedur_ns = 0, framedur_ns = 0;
615 0 : int dotclock = mode->crtc_clock;
616 :
617 0 : if (!drm_dev_has_vblank(dev))
618 : return;
619 :
620 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
621 : return;
622 :
623 : /* Valid dotclock? */
624 0 : if (dotclock > 0) {
625 0 : int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
626 :
627 : /*
628 : * Convert scanline length in pixels and video
629 : * dot clock to line duration and frame duration
630 : * in nanoseconds:
631 : */
632 0 : linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
633 0 : framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
634 :
635 : /*
636 : * Fields of interlaced scanout modes are only half a frame duration.
637 : */
638 0 : if (mode->flags & DRM_MODE_FLAG_INTERLACE)
639 0 : framedur_ns /= 2;
640 : } else {
641 0 : drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
642 : crtc->base.id);
643 : }
644 :
645 0 : vblank->linedur_ns = linedur_ns;
646 0 : vblank->framedur_ns = framedur_ns;
647 0 : drm_mode_copy(&vblank->hwmode, mode);
648 :
649 0 : drm_dbg_core(dev,
650 : "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
651 : crtc->base.id, mode->crtc_htotal,
652 : mode->crtc_vtotal, mode->crtc_vdisplay);
653 0 : drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
654 : crtc->base.id, dotclock, framedur_ns, linedur_ns);
655 : }
656 : EXPORT_SYMBOL(drm_calc_timestamping_constants);
657 :
658 : /**
659 : * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
660 : * timestamp helper
661 : * @crtc: CRTC whose vblank timestamp to retrieve
662 : * @max_error: Desired maximum allowable error in timestamps (nanosecs)
663 : * On return contains true maximum error of timestamp
664 : * @vblank_time: Pointer to time which should receive the timestamp
665 : * @in_vblank_irq:
666 : * True when called from drm_crtc_handle_vblank(). Some drivers
667 : * need to apply some workarounds for gpu-specific vblank irq quirks
668 : * if flag is set.
669 : * @get_scanout_position:
670 : * Callback function to retrieve the scanout position. See
671 : * @struct drm_crtc_helper_funcs.get_scanout_position.
672 : *
673 : * Implements calculation of exact vblank timestamps from given drm_display_mode
674 : * timings and current video scanout position of a CRTC.
675 : *
676 : * The current implementation only handles standard video modes. For double scan
677 : * and interlaced modes the driver is supposed to adjust the hardware mode
678 : * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
679 : * match the scanout position reported.
680 : *
681 : * Note that atomic drivers must call drm_calc_timestamping_constants() before
682 : * enabling a CRTC. The atomic helpers already take care of that in
683 : * drm_atomic_helper_calc_timestamping_constants().
684 : *
685 : * Returns:
686 : *
687 : * Returns true on success, and false on failure, i.e. when no accurate
688 : * timestamp could be acquired.
689 : */
690 : bool
691 0 : drm_crtc_vblank_helper_get_vblank_timestamp_internal(
692 : struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
693 : bool in_vblank_irq,
694 : drm_vblank_get_scanout_position_func get_scanout_position)
695 : {
696 0 : struct drm_device *dev = crtc->dev;
697 0 : unsigned int pipe = crtc->index;
698 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
699 : struct timespec64 ts_etime, ts_vblank_time;
700 : ktime_t stime, etime;
701 : bool vbl_status;
702 : const struct drm_display_mode *mode;
703 : int vpos, hpos, i;
704 : int delta_ns, duration_ns;
705 :
706 0 : if (pipe >= dev->num_crtcs) {
707 0 : drm_err(dev, "Invalid crtc %u\n", pipe);
708 0 : return false;
709 : }
710 :
711 : /* Scanout position query not supported? Should not happen. */
712 0 : if (!get_scanout_position) {
713 0 : drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
714 0 : return false;
715 : }
716 :
717 0 : if (drm_drv_uses_atomic_modeset(dev))
718 0 : mode = &vblank->hwmode;
719 : else
720 0 : mode = &crtc->hwmode;
721 :
722 : /* If mode timing undefined, just return as no-op:
723 : * Happens during initial modesetting of a crtc.
724 : */
725 0 : if (mode->crtc_clock == 0) {
726 0 : drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
727 : pipe);
728 0 : drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
729 : return false;
730 : }
731 :
732 : /* Get current scanout position with system timestamp.
733 : * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
734 : * if single query takes longer than max_error nanoseconds.
735 : *
736 : * This guarantees a tight bound on maximum error if
737 : * code gets preempted or delayed for some reason.
738 : */
739 0 : for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
740 : /*
741 : * Get vertical and horizontal scanout position vpos, hpos,
742 : * and bounding timestamps stime, etime, pre/post query.
743 : */
744 0 : vbl_status = get_scanout_position(crtc, in_vblank_irq,
745 : &vpos, &hpos,
746 : &stime, &etime,
747 : mode);
748 :
749 : /* Return as no-op if scanout query unsupported or failed. */
750 0 : if (!vbl_status) {
751 0 : drm_dbg_core(dev,
752 : "crtc %u : scanoutpos query failed.\n",
753 : pipe);
754 0 : return false;
755 : }
756 :
757 : /* Compute uncertainty in timestamp of scanout position query. */
758 0 : duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
759 :
760 : /* Accept result with < max_error nsecs timing uncertainty. */
761 0 : if (duration_ns <= *max_error)
762 : break;
763 : }
764 :
765 : /* Noisy system timing? */
766 0 : if (i == DRM_TIMESTAMP_MAXRETRIES) {
767 0 : drm_dbg_core(dev,
768 : "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
769 : pipe, duration_ns / 1000, *max_error / 1000, i);
770 : }
771 :
772 : /* Return upper bound of timestamp precision error. */
773 0 : *max_error = duration_ns;
774 :
775 : /* Convert scanout position into elapsed time at raw_time query
776 : * since start of scanout at first display scanline. delta_ns
777 : * can be negative if start of scanout hasn't happened yet.
778 : */
779 0 : delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
780 : mode->crtc_clock);
781 :
782 : /* Subtract time delta from raw timestamp to get final
783 : * vblank_time timestamp for end of vblank.
784 : */
785 0 : *vblank_time = ktime_sub_ns(etime, delta_ns);
786 :
787 0 : if (!drm_debug_enabled(DRM_UT_VBL))
788 : return true;
789 :
790 0 : ts_etime = ktime_to_timespec64(etime);
791 0 : ts_vblank_time = ktime_to_timespec64(*vblank_time);
792 :
793 0 : drm_dbg_vbl(dev,
794 : "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
795 : pipe, hpos, vpos,
796 : (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
797 : (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
798 : duration_ns / 1000, i);
799 :
800 0 : return true;
801 : }
802 : EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
803 :
804 : /**
805 : * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
806 : * helper
807 : * @crtc: CRTC whose vblank timestamp to retrieve
808 : * @max_error: Desired maximum allowable error in timestamps (nanosecs)
809 : * On return contains true maximum error of timestamp
810 : * @vblank_time: Pointer to time which should receive the timestamp
811 : * @in_vblank_irq:
812 : * True when called from drm_crtc_handle_vblank(). Some drivers
813 : * need to apply some workarounds for gpu-specific vblank irq quirks
814 : * if flag is set.
815 : *
816 : * Implements calculation of exact vblank timestamps from given drm_display_mode
817 : * timings and current video scanout position of a CRTC. This can be directly
818 : * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
819 : * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
820 : *
821 : * The current implementation only handles standard video modes. For double scan
822 : * and interlaced modes the driver is supposed to adjust the hardware mode
823 : * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
824 : * match the scanout position reported.
825 : *
826 : * Note that atomic drivers must call drm_calc_timestamping_constants() before
827 : * enabling a CRTC. The atomic helpers already take care of that in
828 : * drm_atomic_helper_calc_timestamping_constants().
829 : *
830 : * Returns:
831 : *
832 : * Returns true on success, and false on failure, i.e. when no accurate
833 : * timestamp could be acquired.
834 : */
835 0 : bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
836 : int *max_error,
837 : ktime_t *vblank_time,
838 : bool in_vblank_irq)
839 : {
840 0 : return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
841 : crtc, max_error, vblank_time, in_vblank_irq,
842 0 : crtc->helper_private->get_scanout_position);
843 : }
844 : EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
845 :
846 : /**
847 : * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most
848 : * recent vblank interval
849 : * @crtc: CRTC whose vblank timestamp to retrieve
850 : * @tvblank: Pointer to target time which should receive the timestamp
851 : * @in_vblank_irq:
852 : * True when called from drm_crtc_handle_vblank(). Some drivers
853 : * need to apply some workarounds for gpu-specific vblank irq quirks
854 : * if flag is set.
855 : *
856 : * Fetches the system timestamp corresponding to the time of the most recent
857 : * vblank interval on specified CRTC. May call into kms-driver to
858 : * compute the timestamp with a high-precision GPU specific method.
859 : *
860 : * Returns zero if timestamp originates from uncorrected do_gettimeofday()
861 : * call, i.e., it isn't very precisely locked to the true vblank.
862 : *
863 : * Returns:
864 : * True if timestamp is considered to be very precise, false otherwise.
865 : */
866 : static bool
867 0 : drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,
868 : bool in_vblank_irq)
869 : {
870 0 : bool ret = false;
871 :
872 : /* Define requested maximum error on timestamps (nanoseconds). */
873 0 : int max_error = (int) drm_timestamp_precision * 1000;
874 :
875 : /* Query driver if possible and precision timestamping enabled. */
876 0 : if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
877 0 : ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
878 : tvblank, in_vblank_irq);
879 : }
880 :
881 : /* GPU high precision timestamp query unsupported or failed.
882 : * Return current monotonic/gettimeofday timestamp as best estimate.
883 : */
884 0 : if (!ret)
885 0 : *tvblank = ktime_get();
886 :
887 0 : return ret;
888 : }
889 :
890 : static bool
891 : drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
892 : ktime_t *tvblank, bool in_vblank_irq)
893 : {
894 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
895 :
896 0 : return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);
897 : }
898 :
899 : /**
900 : * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
901 : * @crtc: which counter to retrieve
902 : *
903 : * Fetches the "cooked" vblank count value that represents the number of
904 : * vblank events since the system was booted, including lost events due to
905 : * modesetting activity. Note that this timer isn't correct against a racing
906 : * vblank interrupt (since it only reports the software vblank counter), see
907 : * drm_crtc_accurate_vblank_count() for such use-cases.
908 : *
909 : * Note that for a given vblank counter value drm_crtc_handle_vblank()
910 : * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
911 : * provide a barrier: Any writes done before calling
912 : * drm_crtc_handle_vblank() will be visible to callers of the later
913 : * functions, if the vblank count is the same or a later one.
914 : *
915 : * See also &drm_vblank_crtc.count.
916 : *
917 : * Returns:
918 : * The software vblank counter.
919 : */
920 0 : u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
921 : {
922 0 : return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
923 : }
924 : EXPORT_SYMBOL(drm_crtc_vblank_count);
925 :
926 : /**
927 : * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
928 : * system timestamp corresponding to that vblank counter value.
929 : * @dev: DRM device
930 : * @pipe: index of CRTC whose counter to retrieve
931 : * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
932 : *
933 : * Fetches the "cooked" vblank count value that represents the number of
934 : * vblank events since the system was booted, including lost events due to
935 : * modesetting activity. Returns corresponding system timestamp of the time
936 : * of the vblank interval that corresponds to the current vblank counter value.
937 : *
938 : * This is the legacy version of drm_crtc_vblank_count_and_time().
939 : */
940 0 : static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
941 : ktime_t *vblanktime)
942 : {
943 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
944 : u64 vblank_count;
945 : unsigned int seq;
946 :
947 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
948 0 : *vblanktime = 0;
949 0 : return 0;
950 : }
951 :
952 : do {
953 0 : seq = read_seqbegin(&vblank->seqlock);
954 0 : vblank_count = atomic64_read(&vblank->count);
955 0 : *vblanktime = vblank->time;
956 0 : } while (read_seqretry(&vblank->seqlock, seq));
957 :
958 : return vblank_count;
959 : }
960 :
961 : /**
962 : * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
963 : * and the system timestamp corresponding to that vblank counter value
964 : * @crtc: which counter to retrieve
965 : * @vblanktime: Pointer to time to receive the vblank timestamp.
966 : *
967 : * Fetches the "cooked" vblank count value that represents the number of
968 : * vblank events since the system was booted, including lost events due to
969 : * modesetting activity. Returns corresponding system timestamp of the time
970 : * of the vblank interval that corresponds to the current vblank counter value.
971 : *
972 : * Note that for a given vblank counter value drm_crtc_handle_vblank()
973 : * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
974 : * provide a barrier: Any writes done before calling
975 : * drm_crtc_handle_vblank() will be visible to callers of the later
976 : * functions, if the vblank count is the same or a later one.
977 : *
978 : * See also &drm_vblank_crtc.count.
979 : */
980 0 : u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
981 : ktime_t *vblanktime)
982 : {
983 0 : return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
984 : vblanktime);
985 : }
986 : EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
987 :
988 : /**
989 : * drm_crtc_next_vblank_start - calculate the time of the next vblank
990 : * @crtc: the crtc for which to calculate next vblank time
991 : * @vblanktime: pointer to time to receive the next vblank timestamp.
992 : *
993 : * Calculate the expected time of the start of the next vblank period,
994 : * based on time of previous vblank and frame duration
995 : */
996 0 : int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
997 : {
998 0 : unsigned int pipe = drm_crtc_index(crtc);
999 : struct drm_vblank_crtc *vblank;
1000 : struct drm_display_mode *mode;
1001 : u64 vblank_start;
1002 :
1003 0 : if (!drm_dev_has_vblank(crtc->dev))
1004 : return -EINVAL;
1005 :
1006 0 : vblank = &crtc->dev->vblank[pipe];
1007 0 : mode = &vblank->hwmode;
1008 :
1009 0 : if (!vblank->framedur_ns || !vblank->linedur_ns)
1010 : return -EINVAL;
1011 :
1012 0 : if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false))
1013 : return -EINVAL;
1014 :
1015 0 : vblank_start = DIV_ROUND_DOWN_ULL(
1016 : (u64)vblank->framedur_ns * mode->crtc_vblank_start,
1017 : mode->crtc_vtotal);
1018 0 : *vblanktime = ktime_add(*vblanktime, ns_to_ktime(vblank_start));
1019 :
1020 0 : return 0;
1021 : }
1022 : EXPORT_SYMBOL(drm_crtc_next_vblank_start);
1023 :
1024 0 : static void send_vblank_event(struct drm_device *dev,
1025 : struct drm_pending_vblank_event *e,
1026 : u64 seq, ktime_t now)
1027 : {
1028 : struct timespec64 tv;
1029 :
1030 0 : switch (e->event.base.type) {
1031 : case DRM_EVENT_VBLANK:
1032 : case DRM_EVENT_FLIP_COMPLETE:
1033 0 : tv = ktime_to_timespec64(now);
1034 0 : e->event.vbl.sequence = seq;
1035 : /*
1036 : * e->event is a user space structure, with hardcoded unsigned
1037 : * 32-bit seconds/microseconds. This is safe as we always use
1038 : * monotonic timestamps since linux-4.15
1039 : */
1040 0 : e->event.vbl.tv_sec = tv.tv_sec;
1041 0 : e->event.vbl.tv_usec = tv.tv_nsec / 1000;
1042 0 : break;
1043 : case DRM_EVENT_CRTC_SEQUENCE:
1044 0 : if (seq)
1045 0 : e->event.seq.sequence = seq;
1046 0 : e->event.seq.time_ns = ktime_to_ns(now);
1047 0 : break;
1048 : }
1049 0 : trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
1050 : /*
1051 : * Use the same timestamp for any associated fence signal to avoid
1052 : * mismatch in timestamps for vsync & fence events triggered by the
1053 : * same HW event. Frameworks like SurfaceFlinger in Android expects the
1054 : * retire-fence timestamp to match exactly with HW vsync as it uses it
1055 : * for its software vsync modeling.
1056 : */
1057 0 : drm_send_event_timestamp_locked(dev, &e->base, now);
1058 0 : }
1059 :
1060 : /**
1061 : * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1062 : * @crtc: the source CRTC of the vblank event
1063 : * @e: the event to send
1064 : *
1065 : * A lot of drivers need to generate vblank events for the very next vblank
1066 : * interrupt. For example when the page flip interrupt happens when the page
1067 : * flip gets armed, but not when it actually executes within the next vblank
1068 : * period. This helper function implements exactly the required vblank arming
1069 : * behaviour.
1070 : *
1071 : * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1072 : * atomic commit must ensure that the next vblank happens at exactly the same
1073 : * time as the atomic commit is committed to the hardware. This function itself
1074 : * does **not** protect against the next vblank interrupt racing with either this
1075 : * function call or the atomic commit operation. A possible sequence could be:
1076 : *
1077 : * 1. Driver commits new hardware state into vblank-synchronized registers.
1078 : * 2. A vblank happens, committing the hardware state. Also the corresponding
1079 : * vblank interrupt is fired off and fully processed by the interrupt
1080 : * handler.
1081 : * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1082 : * 4. The event is only send out for the next vblank, which is wrong.
1083 : *
1084 : * An equivalent race can happen when the driver calls
1085 : * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1086 : *
1087 : * The only way to make this work safely is to prevent the vblank from firing
1088 : * (and the hardware from committing anything else) until the entire atomic
1089 : * commit sequence has run to completion. If the hardware does not have such a
1090 : * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1091 : * Instead drivers need to manually send out the event from their interrupt
1092 : * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1093 : * possible race with the hardware committing the atomic update.
1094 : *
1095 : * Caller must hold a vblank reference for the event @e acquired by a
1096 : * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1097 : */
1098 0 : void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1099 : struct drm_pending_vblank_event *e)
1100 : {
1101 0 : struct drm_device *dev = crtc->dev;
1102 0 : unsigned int pipe = drm_crtc_index(crtc);
1103 :
1104 : assert_spin_locked(&dev->event_lock);
1105 :
1106 0 : e->pipe = pipe;
1107 0 : e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1108 0 : list_add_tail(&e->base.link, &dev->vblank_event_list);
1109 0 : }
1110 : EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1111 :
1112 : /**
1113 : * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1114 : * @crtc: the source CRTC of the vblank event
1115 : * @e: the event to send
1116 : *
1117 : * Updates sequence # and timestamp on event for the most recently processed
1118 : * vblank, and sends it to userspace. Caller must hold event lock.
1119 : *
1120 : * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1121 : * situation, especially to send out events for atomic commit operations.
1122 : */
1123 0 : void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1124 : struct drm_pending_vblank_event *e)
1125 : {
1126 0 : struct drm_device *dev = crtc->dev;
1127 : u64 seq;
1128 0 : unsigned int pipe = drm_crtc_index(crtc);
1129 : ktime_t now;
1130 :
1131 0 : if (drm_dev_has_vblank(dev)) {
1132 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
1133 : } else {
1134 0 : seq = 0;
1135 :
1136 0 : now = ktime_get();
1137 : }
1138 0 : e->pipe = pipe;
1139 0 : send_vblank_event(dev, e, seq, now);
1140 0 : }
1141 : EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1142 :
1143 0 : static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1144 : {
1145 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1146 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1147 :
1148 0 : if (drm_WARN_ON(dev, !crtc))
1149 : return 0;
1150 :
1151 0 : if (crtc->funcs->enable_vblank)
1152 0 : return crtc->funcs->enable_vblank(crtc);
1153 : }
1154 : #ifdef CONFIG_DRM_LEGACY
1155 : else if (dev->driver->enable_vblank) {
1156 : return dev->driver->enable_vblank(dev, pipe);
1157 : }
1158 : #endif
1159 :
1160 : return -EINVAL;
1161 : }
1162 :
1163 0 : static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1164 : {
1165 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1166 0 : int ret = 0;
1167 :
1168 : assert_spin_locked(&dev->vbl_lock);
1169 :
1170 0 : spin_lock(&dev->vblank_time_lock);
1171 :
1172 0 : if (!vblank->enabled) {
1173 : /*
1174 : * Enable vblank irqs under vblank_time_lock protection.
1175 : * All vblank count & timestamp updates are held off
1176 : * until we are done reinitializing master counter and
1177 : * timestamps. Filtercode in drm_handle_vblank() will
1178 : * prevent double-accounting of same vblank interval.
1179 : */
1180 0 : ret = __enable_vblank(dev, pipe);
1181 0 : drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1182 : pipe, ret);
1183 0 : if (ret) {
1184 0 : atomic_dec(&vblank->refcount);
1185 : } else {
1186 0 : drm_update_vblank_count(dev, pipe, 0);
1187 : /* drm_update_vblank_count() includes a wmb so we just
1188 : * need to ensure that the compiler emits the write
1189 : * to mark the vblank as enabled after the call
1190 : * to drm_update_vblank_count().
1191 : */
1192 0 : WRITE_ONCE(vblank->enabled, true);
1193 : }
1194 : }
1195 :
1196 0 : spin_unlock(&dev->vblank_time_lock);
1197 :
1198 0 : return ret;
1199 : }
1200 :
1201 0 : int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1202 : {
1203 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1204 : unsigned long irqflags;
1205 0 : int ret = 0;
1206 :
1207 0 : if (!drm_dev_has_vblank(dev))
1208 : return -EINVAL;
1209 :
1210 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1211 : return -EINVAL;
1212 :
1213 0 : spin_lock_irqsave(&dev->vbl_lock, irqflags);
1214 : /* Going from 0->1 means we have to enable interrupts again */
1215 0 : if (atomic_add_return(1, &vblank->refcount) == 1) {
1216 0 : ret = drm_vblank_enable(dev, pipe);
1217 : } else {
1218 0 : if (!vblank->enabled) {
1219 0 : atomic_dec(&vblank->refcount);
1220 0 : ret = -EINVAL;
1221 : }
1222 : }
1223 0 : spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1224 :
1225 0 : return ret;
1226 : }
1227 :
1228 : /**
1229 : * drm_crtc_vblank_get - get a reference count on vblank events
1230 : * @crtc: which CRTC to own
1231 : *
1232 : * Acquire a reference count on vblank events to avoid having them disabled
1233 : * while in use.
1234 : *
1235 : * Returns:
1236 : * Zero on success or a negative error code on failure.
1237 : */
1238 0 : int drm_crtc_vblank_get(struct drm_crtc *crtc)
1239 : {
1240 0 : return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1241 : }
1242 : EXPORT_SYMBOL(drm_crtc_vblank_get);
1243 :
1244 0 : void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1245 : {
1246 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1247 :
1248 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1249 : return;
1250 :
1251 0 : if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1252 : return;
1253 :
1254 : /* Last user schedules interrupt disable */
1255 0 : if (atomic_dec_and_test(&vblank->refcount)) {
1256 0 : if (drm_vblank_offdelay == 0)
1257 : return;
1258 0 : else if (drm_vblank_offdelay < 0)
1259 0 : vblank_disable_fn(&vblank->disable_timer);
1260 0 : else if (!dev->vblank_disable_immediate)
1261 0 : mod_timer(&vblank->disable_timer,
1262 0 : jiffies + ((drm_vblank_offdelay * HZ)/1000));
1263 : }
1264 : }
1265 :
1266 : /**
1267 : * drm_crtc_vblank_put - give up ownership of vblank events
1268 : * @crtc: which counter to give up
1269 : *
1270 : * Release ownership of a given vblank counter, turning off interrupts
1271 : * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1272 : */
1273 0 : void drm_crtc_vblank_put(struct drm_crtc *crtc)
1274 : {
1275 0 : drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1276 0 : }
1277 : EXPORT_SYMBOL(drm_crtc_vblank_put);
1278 :
1279 : /**
1280 : * drm_wait_one_vblank - wait for one vblank
1281 : * @dev: DRM device
1282 : * @pipe: CRTC index
1283 : *
1284 : * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1285 : * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1286 : * due to lack of driver support or because the crtc is off.
1287 : *
1288 : * This is the legacy version of drm_crtc_wait_one_vblank().
1289 : */
1290 0 : void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1291 : {
1292 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1293 : int ret;
1294 : u64 last;
1295 :
1296 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1297 : return;
1298 :
1299 0 : ret = drm_vblank_get(dev, pipe);
1300 0 : if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1301 : pipe, ret))
1302 : return;
1303 :
1304 0 : last = drm_vblank_count(dev, pipe);
1305 :
1306 0 : ret = wait_event_timeout(vblank->queue,
1307 : last != drm_vblank_count(dev, pipe),
1308 : msecs_to_jiffies(100));
1309 :
1310 0 : drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1311 :
1312 0 : drm_vblank_put(dev, pipe);
1313 : }
1314 : EXPORT_SYMBOL(drm_wait_one_vblank);
1315 :
1316 : /**
1317 : * drm_crtc_wait_one_vblank - wait for one vblank
1318 : * @crtc: DRM crtc
1319 : *
1320 : * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1321 : * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1322 : * due to lack of driver support or because the crtc is off.
1323 : */
1324 0 : void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1325 : {
1326 0 : drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1327 0 : }
1328 : EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1329 :
1330 : /**
1331 : * drm_crtc_vblank_off - disable vblank events on a CRTC
1332 : * @crtc: CRTC in question
1333 : *
1334 : * Drivers can use this function to shut down the vblank interrupt handling when
1335 : * disabling a crtc. This function ensures that the latest vblank frame count is
1336 : * stored so that drm_vblank_on can restore it again.
1337 : *
1338 : * Drivers must use this function when the hardware vblank counter can get
1339 : * reset, e.g. when suspending or disabling the @crtc in general.
1340 : */
1341 0 : void drm_crtc_vblank_off(struct drm_crtc *crtc)
1342 : {
1343 0 : struct drm_device *dev = crtc->dev;
1344 0 : unsigned int pipe = drm_crtc_index(crtc);
1345 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1346 : struct drm_pending_vblank_event *e, *t;
1347 : ktime_t now;
1348 : u64 seq;
1349 :
1350 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1351 0 : return;
1352 :
1353 : /*
1354 : * Grab event_lock early to prevent vblank work from being scheduled
1355 : * while we're in the middle of shutting down vblank interrupts
1356 : */
1357 0 : spin_lock_irq(&dev->event_lock);
1358 :
1359 0 : spin_lock(&dev->vbl_lock);
1360 0 : drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1361 : pipe, vblank->enabled, vblank->inmodeset);
1362 :
1363 : /* Avoid redundant vblank disables without previous
1364 : * drm_crtc_vblank_on(). */
1365 0 : if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1366 0 : drm_vblank_disable_and_save(dev, pipe);
1367 :
1368 0 : wake_up(&vblank->queue);
1369 :
1370 : /*
1371 : * Prevent subsequent drm_vblank_get() from re-enabling
1372 : * the vblank interrupt by bumping the refcount.
1373 : */
1374 0 : if (!vblank->inmodeset) {
1375 0 : atomic_inc(&vblank->refcount);
1376 0 : vblank->inmodeset = 1;
1377 : }
1378 0 : spin_unlock(&dev->vbl_lock);
1379 :
1380 : /* Send any queued vblank events, lest the natives grow disquiet */
1381 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
1382 :
1383 0 : list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1384 0 : if (e->pipe != pipe)
1385 0 : continue;
1386 0 : drm_dbg_core(dev, "Sending premature vblank event on disable: "
1387 : "wanted %llu, current %llu\n",
1388 : e->sequence, seq);
1389 0 : list_del(&e->base.link);
1390 0 : drm_vblank_put(dev, pipe);
1391 0 : send_vblank_event(dev, e, seq, now);
1392 : }
1393 :
1394 : /* Cancel any leftover pending vblank work */
1395 0 : drm_vblank_cancel_pending_works(vblank);
1396 :
1397 0 : spin_unlock_irq(&dev->event_lock);
1398 :
1399 : /* Will be reset by the modeset helpers when re-enabling the crtc by
1400 : * calling drm_calc_timestamping_constants(). */
1401 0 : vblank->hwmode.crtc_clock = 0;
1402 :
1403 : /* Wait for any vblank work that's still executing to finish */
1404 0 : drm_vblank_flush_worker(vblank);
1405 : }
1406 : EXPORT_SYMBOL(drm_crtc_vblank_off);
1407 :
1408 : /**
1409 : * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1410 : * @crtc: CRTC in question
1411 : *
1412 : * Drivers can use this function to reset the vblank state to off at load time.
1413 : * Drivers should use this together with the drm_crtc_vblank_off() and
1414 : * drm_crtc_vblank_on() functions. The difference compared to
1415 : * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1416 : * and hence doesn't need to call any driver hooks.
1417 : *
1418 : * This is useful for recovering driver state e.g. on driver load, or on resume.
1419 : */
1420 0 : void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1421 : {
1422 0 : struct drm_device *dev = crtc->dev;
1423 0 : unsigned int pipe = drm_crtc_index(crtc);
1424 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1425 :
1426 0 : spin_lock_irq(&dev->vbl_lock);
1427 : /*
1428 : * Prevent subsequent drm_vblank_get() from enabling the vblank
1429 : * interrupt by bumping the refcount.
1430 : */
1431 0 : if (!vblank->inmodeset) {
1432 0 : atomic_inc(&vblank->refcount);
1433 0 : vblank->inmodeset = 1;
1434 : }
1435 0 : spin_unlock_irq(&dev->vbl_lock);
1436 :
1437 0 : drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1438 0 : drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
1439 0 : }
1440 : EXPORT_SYMBOL(drm_crtc_vblank_reset);
1441 :
1442 : /**
1443 : * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1444 : * @crtc: CRTC in question
1445 : * @max_vblank_count: max hardware vblank counter value
1446 : *
1447 : * Update the maximum hardware vblank counter value for @crtc
1448 : * at runtime. Useful for hardware where the operation of the
1449 : * hardware vblank counter depends on the currently active
1450 : * display configuration.
1451 : *
1452 : * For example, if the hardware vblank counter does not work
1453 : * when a specific connector is active the maximum can be set
1454 : * to zero. And when that specific connector isn't active the
1455 : * maximum can again be set to the appropriate non-zero value.
1456 : *
1457 : * If used, must be called before drm_vblank_on().
1458 : */
1459 0 : void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1460 : u32 max_vblank_count)
1461 : {
1462 0 : struct drm_device *dev = crtc->dev;
1463 0 : unsigned int pipe = drm_crtc_index(crtc);
1464 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1465 :
1466 0 : drm_WARN_ON(dev, dev->max_vblank_count);
1467 0 : drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1468 :
1469 0 : vblank->max_vblank_count = max_vblank_count;
1470 0 : }
1471 : EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1472 :
1473 : /**
1474 : * drm_crtc_vblank_on - enable vblank events on a CRTC
1475 : * @crtc: CRTC in question
1476 : *
1477 : * This functions restores the vblank interrupt state captured with
1478 : * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1479 : * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1480 : * unbalanced and so can also be unconditionally called in driver load code to
1481 : * reflect the current hardware state of the crtc.
1482 : */
1483 0 : void drm_crtc_vblank_on(struct drm_crtc *crtc)
1484 : {
1485 0 : struct drm_device *dev = crtc->dev;
1486 0 : unsigned int pipe = drm_crtc_index(crtc);
1487 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1488 :
1489 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1490 : return;
1491 :
1492 0 : spin_lock_irq(&dev->vbl_lock);
1493 0 : drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1494 : pipe, vblank->enabled, vblank->inmodeset);
1495 :
1496 : /* Drop our private "prevent drm_vblank_get" refcount */
1497 0 : if (vblank->inmodeset) {
1498 0 : atomic_dec(&vblank->refcount);
1499 0 : vblank->inmodeset = 0;
1500 : }
1501 :
1502 0 : drm_reset_vblank_timestamp(dev, pipe);
1503 :
1504 : /*
1505 : * re-enable interrupts if there are users left, or the
1506 : * user wishes vblank interrupts to be enabled all the time.
1507 : */
1508 0 : if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1509 0 : drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1510 0 : spin_unlock_irq(&dev->vbl_lock);
1511 : }
1512 : EXPORT_SYMBOL(drm_crtc_vblank_on);
1513 :
1514 0 : static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1515 : {
1516 : ktime_t t_vblank;
1517 : struct drm_vblank_crtc *vblank;
1518 : int framedur_ns;
1519 : u64 diff_ns;
1520 0 : u32 cur_vblank, diff = 1;
1521 0 : int count = DRM_TIMESTAMP_MAXRETRIES;
1522 0 : u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
1523 :
1524 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1525 0 : return;
1526 :
1527 : assert_spin_locked(&dev->vbl_lock);
1528 : assert_spin_locked(&dev->vblank_time_lock);
1529 :
1530 0 : vblank = &dev->vblank[pipe];
1531 0 : drm_WARN_ONCE(dev,
1532 : drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1533 : "Cannot compute missed vblanks without frame duration\n");
1534 0 : framedur_ns = vblank->framedur_ns;
1535 :
1536 : do {
1537 0 : cur_vblank = __get_vblank_counter(dev, pipe);
1538 0 : drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1539 0 : } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1540 :
1541 0 : diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1542 0 : if (framedur_ns)
1543 0 : diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1544 :
1545 :
1546 0 : drm_dbg_vbl(dev,
1547 : "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1548 : diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1549 0 : vblank->last = (cur_vblank - diff) & max_vblank_count;
1550 : }
1551 :
1552 : /**
1553 : * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1554 : * @crtc: CRTC in question
1555 : *
1556 : * Power manamement features can cause frame counter resets between vblank
1557 : * disable and enable. Drivers can use this function in their
1558 : * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1559 : * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1560 : * vblank counter.
1561 : *
1562 : * Note that drivers must have race-free high-precision timestamping support,
1563 : * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
1564 : * &drm_driver.vblank_disable_immediate must be set to indicate the
1565 : * time-stamping functions are race-free against vblank hardware counter
1566 : * increments.
1567 : */
1568 0 : void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1569 : {
1570 0 : WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp);
1571 0 : WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate);
1572 :
1573 0 : drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1574 0 : }
1575 : EXPORT_SYMBOL(drm_crtc_vblank_restore);
1576 :
1577 0 : static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1578 : unsigned int pipe)
1579 : {
1580 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1581 :
1582 : /* vblank is not initialized (IRQ not installed ?), or has been freed */
1583 0 : if (!drm_dev_has_vblank(dev))
1584 : return;
1585 :
1586 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1587 : return;
1588 :
1589 : /*
1590 : * To avoid all the problems that might happen if interrupts
1591 : * were enabled/disabled around or between these calls, we just
1592 : * have the kernel take a reference on the CRTC (just once though
1593 : * to avoid corrupting the count if multiple, mismatch calls occur),
1594 : * so that interrupts remain enabled in the interim.
1595 : */
1596 0 : if (!vblank->inmodeset) {
1597 0 : vblank->inmodeset = 0x1;
1598 0 : if (drm_vblank_get(dev, pipe) == 0)
1599 0 : vblank->inmodeset |= 0x2;
1600 : }
1601 : }
1602 :
1603 0 : static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1604 : unsigned int pipe)
1605 : {
1606 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1607 :
1608 : /* vblank is not initialized (IRQ not installed ?), or has been freed */
1609 0 : if (!drm_dev_has_vblank(dev))
1610 : return;
1611 :
1612 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1613 : return;
1614 :
1615 0 : if (vblank->inmodeset) {
1616 0 : spin_lock_irq(&dev->vbl_lock);
1617 0 : drm_reset_vblank_timestamp(dev, pipe);
1618 0 : spin_unlock_irq(&dev->vbl_lock);
1619 :
1620 0 : if (vblank->inmodeset & 0x2)
1621 0 : drm_vblank_put(dev, pipe);
1622 :
1623 0 : vblank->inmodeset = 0;
1624 : }
1625 : }
1626 :
1627 0 : int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1628 : struct drm_file *file_priv)
1629 : {
1630 0 : struct drm_modeset_ctl *modeset = data;
1631 : unsigned int pipe;
1632 :
1633 : /* If drm_vblank_init() hasn't been called yet, just no-op */
1634 0 : if (!drm_dev_has_vblank(dev))
1635 : return 0;
1636 :
1637 : /* KMS drivers handle this internally */
1638 0 : if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1639 : return 0;
1640 :
1641 0 : pipe = modeset->crtc;
1642 0 : if (pipe >= dev->num_crtcs)
1643 : return -EINVAL;
1644 :
1645 0 : switch (modeset->cmd) {
1646 : case _DRM_PRE_MODESET:
1647 0 : drm_legacy_vblank_pre_modeset(dev, pipe);
1648 0 : break;
1649 : case _DRM_POST_MODESET:
1650 0 : drm_legacy_vblank_post_modeset(dev, pipe);
1651 0 : break;
1652 : default:
1653 : return -EINVAL;
1654 : }
1655 :
1656 : return 0;
1657 : }
1658 :
1659 0 : static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1660 : u64 req_seq,
1661 : union drm_wait_vblank *vblwait,
1662 : struct drm_file *file_priv)
1663 : {
1664 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1665 : struct drm_pending_vblank_event *e;
1666 : ktime_t now;
1667 : u64 seq;
1668 : int ret;
1669 :
1670 0 : e = kzalloc(sizeof(*e), GFP_KERNEL);
1671 0 : if (e == NULL) {
1672 : ret = -ENOMEM;
1673 : goto err_put;
1674 : }
1675 :
1676 0 : e->pipe = pipe;
1677 0 : e->event.base.type = DRM_EVENT_VBLANK;
1678 0 : e->event.base.length = sizeof(e->event.vbl);
1679 0 : e->event.vbl.user_data = vblwait->request.signal;
1680 0 : e->event.vbl.crtc_id = 0;
1681 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1682 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1683 :
1684 0 : if (crtc)
1685 0 : e->event.vbl.crtc_id = crtc->base.id;
1686 : }
1687 :
1688 0 : spin_lock_irq(&dev->event_lock);
1689 :
1690 : /*
1691 : * drm_crtc_vblank_off() might have been called after we called
1692 : * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1693 : * vblank disable, so no need for further locking. The reference from
1694 : * drm_vblank_get() protects against vblank disable from another source.
1695 : */
1696 0 : if (!READ_ONCE(vblank->enabled)) {
1697 : ret = -EINVAL;
1698 : goto err_unlock;
1699 : }
1700 :
1701 0 : ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1702 : &e->event.base);
1703 :
1704 0 : if (ret)
1705 : goto err_unlock;
1706 :
1707 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
1708 :
1709 0 : drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1710 : req_seq, seq, pipe);
1711 :
1712 0 : trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1713 :
1714 0 : e->sequence = req_seq;
1715 0 : if (drm_vblank_passed(seq, req_seq)) {
1716 0 : drm_vblank_put(dev, pipe);
1717 0 : send_vblank_event(dev, e, seq, now);
1718 0 : vblwait->reply.sequence = seq;
1719 : } else {
1720 : /* drm_handle_vblank_events will call drm_vblank_put */
1721 0 : list_add_tail(&e->base.link, &dev->vblank_event_list);
1722 0 : vblwait->reply.sequence = req_seq;
1723 : }
1724 :
1725 0 : spin_unlock_irq(&dev->event_lock);
1726 :
1727 : return 0;
1728 :
1729 : err_unlock:
1730 0 : spin_unlock_irq(&dev->event_lock);
1731 0 : kfree(e);
1732 : err_put:
1733 0 : drm_vblank_put(dev, pipe);
1734 : return ret;
1735 : }
1736 :
1737 : static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1738 : {
1739 0 : if (vblwait->request.sequence)
1740 : return false;
1741 :
1742 0 : return _DRM_VBLANK_RELATIVE ==
1743 0 : (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1744 : _DRM_VBLANK_EVENT |
1745 : _DRM_VBLANK_NEXTONMISS));
1746 : }
1747 :
1748 : /*
1749 : * Widen a 32-bit param to 64-bits.
1750 : *
1751 : * \param narrow 32-bit value (missing upper 32 bits)
1752 : * \param near 64-bit value that should be 'close' to near
1753 : *
1754 : * This function returns a 64-bit value using the lower 32-bits from
1755 : * 'narrow' and constructing the upper 32-bits so that the result is
1756 : * as close as possible to 'near'.
1757 : */
1758 :
1759 : static u64 widen_32_to_64(u32 narrow, u64 near)
1760 : {
1761 0 : return near + (s32) (narrow - near);
1762 : }
1763 :
1764 0 : static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1765 : struct drm_wait_vblank_reply *reply)
1766 : {
1767 : ktime_t now;
1768 : struct timespec64 ts;
1769 :
1770 : /*
1771 : * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1772 : * to store the seconds. This is safe as we always use monotonic
1773 : * timestamps since linux-4.15.
1774 : */
1775 0 : reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1776 0 : ts = ktime_to_timespec64(now);
1777 0 : reply->tval_sec = (u32)ts.tv_sec;
1778 0 : reply->tval_usec = ts.tv_nsec / 1000;
1779 0 : }
1780 :
1781 : static bool drm_wait_vblank_supported(struct drm_device *dev)
1782 : {
1783 : #if IS_ENABLED(CONFIG_DRM_LEGACY)
1784 : if (unlikely(drm_core_check_feature(dev, DRIVER_LEGACY)))
1785 : return dev->irq_enabled;
1786 : #endif
1787 0 : return drm_dev_has_vblank(dev);
1788 : }
1789 :
1790 0 : int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1791 : struct drm_file *file_priv)
1792 : {
1793 : struct drm_crtc *crtc;
1794 : struct drm_vblank_crtc *vblank;
1795 0 : union drm_wait_vblank *vblwait = data;
1796 : int ret;
1797 : u64 req_seq, seq;
1798 : unsigned int pipe_index;
1799 : unsigned int flags, pipe, high_pipe;
1800 :
1801 0 : if (!drm_wait_vblank_supported(dev))
1802 : return -EOPNOTSUPP;
1803 :
1804 0 : if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1805 : return -EINVAL;
1806 :
1807 0 : if (vblwait->request.type &
1808 : ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1809 : _DRM_VBLANK_HIGH_CRTC_MASK)) {
1810 0 : drm_dbg_core(dev,
1811 : "Unsupported type value 0x%x, supported mask 0x%x\n",
1812 : vblwait->request.type,
1813 : (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1814 : _DRM_VBLANK_HIGH_CRTC_MASK));
1815 0 : return -EINVAL;
1816 : }
1817 :
1818 0 : flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1819 0 : high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1820 0 : if (high_pipe)
1821 0 : pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1822 : else
1823 0 : pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1824 :
1825 : /* Convert lease-relative crtc index into global crtc index */
1826 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1827 0 : pipe = 0;
1828 0 : drm_for_each_crtc(crtc, dev) {
1829 0 : if (drm_lease_held(file_priv, crtc->base.id)) {
1830 0 : if (pipe_index == 0)
1831 : break;
1832 0 : pipe_index--;
1833 : }
1834 0 : pipe++;
1835 : }
1836 : } else {
1837 : pipe = pipe_index;
1838 : }
1839 :
1840 0 : if (pipe >= dev->num_crtcs)
1841 : return -EINVAL;
1842 :
1843 0 : vblank = &dev->vblank[pipe];
1844 :
1845 : /* If the counter is currently enabled and accurate, short-circuit
1846 : * queries to return the cached timestamp of the last vblank.
1847 : */
1848 0 : if (dev->vblank_disable_immediate &&
1849 0 : drm_wait_vblank_is_query(vblwait) &&
1850 0 : READ_ONCE(vblank->enabled)) {
1851 0 : drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1852 0 : return 0;
1853 : }
1854 :
1855 0 : ret = drm_vblank_get(dev, pipe);
1856 0 : if (ret) {
1857 0 : drm_dbg_core(dev,
1858 : "crtc %d failed to acquire vblank counter, %d\n",
1859 : pipe, ret);
1860 0 : return ret;
1861 : }
1862 0 : seq = drm_vblank_count(dev, pipe);
1863 :
1864 0 : switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1865 : case _DRM_VBLANK_RELATIVE:
1866 0 : req_seq = seq + vblwait->request.sequence;
1867 0 : vblwait->request.sequence = req_seq;
1868 0 : vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1869 0 : break;
1870 : case _DRM_VBLANK_ABSOLUTE:
1871 0 : req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1872 0 : break;
1873 : default:
1874 : ret = -EINVAL;
1875 : goto done;
1876 : }
1877 :
1878 0 : if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1879 0 : drm_vblank_passed(seq, req_seq)) {
1880 0 : req_seq = seq + 1;
1881 0 : vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1882 0 : vblwait->request.sequence = req_seq;
1883 : }
1884 :
1885 0 : if (flags & _DRM_VBLANK_EVENT) {
1886 : /* must hold on to the vblank ref until the event fires
1887 : * drm_vblank_put will be called asynchronously
1888 : */
1889 0 : return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1890 : }
1891 :
1892 0 : if (req_seq != seq) {
1893 : int wait;
1894 :
1895 0 : drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1896 : req_seq, pipe);
1897 0 : wait = wait_event_interruptible_timeout(vblank->queue,
1898 : drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1899 : !READ_ONCE(vblank->enabled),
1900 : msecs_to_jiffies(3000));
1901 :
1902 0 : switch (wait) {
1903 : case 0:
1904 : /* timeout */
1905 : ret = -EBUSY;
1906 : break;
1907 : case -ERESTARTSYS:
1908 : /* interrupted by signal */
1909 0 : ret = -EINTR;
1910 0 : break;
1911 : default:
1912 0 : ret = 0;
1913 0 : break;
1914 : }
1915 : }
1916 :
1917 0 : if (ret != -EINTR) {
1918 0 : drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1919 :
1920 0 : drm_dbg_core(dev, "crtc %d returning %u to client\n",
1921 : pipe, vblwait->reply.sequence);
1922 : } else {
1923 0 : drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1924 : pipe);
1925 : }
1926 :
1927 : done:
1928 0 : drm_vblank_put(dev, pipe);
1929 0 : return ret;
1930 : }
1931 :
1932 0 : static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1933 : {
1934 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1935 0 : bool high_prec = false;
1936 : struct drm_pending_vblank_event *e, *t;
1937 : ktime_t now;
1938 : u64 seq;
1939 :
1940 : assert_spin_locked(&dev->event_lock);
1941 :
1942 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
1943 :
1944 0 : list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1945 0 : if (e->pipe != pipe)
1946 0 : continue;
1947 0 : if (!drm_vblank_passed(seq, e->sequence))
1948 0 : continue;
1949 :
1950 0 : drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1951 : e->sequence, seq);
1952 :
1953 0 : list_del(&e->base.link);
1954 0 : drm_vblank_put(dev, pipe);
1955 0 : send_vblank_event(dev, e, seq, now);
1956 : }
1957 :
1958 : if (crtc && crtc->funcs->get_vblank_timestamp)
1959 : high_prec = true;
1960 :
1961 0 : trace_drm_vblank_event(pipe, seq, now, high_prec);
1962 0 : }
1963 :
1964 : /**
1965 : * drm_handle_vblank - handle a vblank event
1966 : * @dev: DRM device
1967 : * @pipe: index of CRTC where this event occurred
1968 : *
1969 : * Drivers should call this routine in their vblank interrupt handlers to
1970 : * update the vblank counter and send any signals that may be pending.
1971 : *
1972 : * This is the legacy version of drm_crtc_handle_vblank().
1973 : */
1974 0 : bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1975 : {
1976 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1977 : unsigned long irqflags;
1978 : bool disable_irq;
1979 :
1980 0 : if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1981 : return false;
1982 :
1983 0 : if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1984 : return false;
1985 :
1986 0 : spin_lock_irqsave(&dev->event_lock, irqflags);
1987 :
1988 : /* Need timestamp lock to prevent concurrent execution with
1989 : * vblank enable/disable, as this would cause inconsistent
1990 : * or corrupted timestamps and vblank counts.
1991 : */
1992 0 : spin_lock(&dev->vblank_time_lock);
1993 :
1994 : /* Vblank irq handling disabled. Nothing to do. */
1995 0 : if (!vblank->enabled) {
1996 0 : spin_unlock(&dev->vblank_time_lock);
1997 0 : spin_unlock_irqrestore(&dev->event_lock, irqflags);
1998 0 : return false;
1999 : }
2000 :
2001 0 : drm_update_vblank_count(dev, pipe, true);
2002 :
2003 0 : spin_unlock(&dev->vblank_time_lock);
2004 :
2005 0 : wake_up(&vblank->queue);
2006 :
2007 : /* With instant-off, we defer disabling the interrupt until after
2008 : * we finish processing the following vblank after all events have
2009 : * been signaled. The disable has to be last (after
2010 : * drm_handle_vblank_events) so that the timestamp is always accurate.
2011 : */
2012 0 : disable_irq = (dev->vblank_disable_immediate &&
2013 0 : drm_vblank_offdelay > 0 &&
2014 0 : !atomic_read(&vblank->refcount));
2015 :
2016 0 : drm_handle_vblank_events(dev, pipe);
2017 0 : drm_handle_vblank_works(vblank);
2018 :
2019 0 : spin_unlock_irqrestore(&dev->event_lock, irqflags);
2020 :
2021 0 : if (disable_irq)
2022 0 : vblank_disable_fn(&vblank->disable_timer);
2023 :
2024 : return true;
2025 : }
2026 : EXPORT_SYMBOL(drm_handle_vblank);
2027 :
2028 : /**
2029 : * drm_crtc_handle_vblank - handle a vblank event
2030 : * @crtc: where this event occurred
2031 : *
2032 : * Drivers should call this routine in their vblank interrupt handlers to
2033 : * update the vblank counter and send any signals that may be pending.
2034 : *
2035 : * This is the native KMS version of drm_handle_vblank().
2036 : *
2037 : * Note that for a given vblank counter value drm_crtc_handle_vblank()
2038 : * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
2039 : * provide a barrier: Any writes done before calling
2040 : * drm_crtc_handle_vblank() will be visible to callers of the later
2041 : * functions, if the vblank count is the same or a later one.
2042 : *
2043 : * See also &drm_vblank_crtc.count.
2044 : *
2045 : * Returns:
2046 : * True if the event was successfully handled, false on failure.
2047 : */
2048 0 : bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
2049 : {
2050 0 : return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
2051 : }
2052 : EXPORT_SYMBOL(drm_crtc_handle_vblank);
2053 :
2054 : /*
2055 : * Get crtc VBLANK count.
2056 : *
2057 : * \param dev DRM device
2058 : * \param data user argument, pointing to a drm_crtc_get_sequence structure.
2059 : * \param file_priv drm file private for the user's open file descriptor
2060 : */
2061 :
2062 0 : int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
2063 : struct drm_file *file_priv)
2064 : {
2065 : struct drm_crtc *crtc;
2066 : struct drm_vblank_crtc *vblank;
2067 : int pipe;
2068 0 : struct drm_crtc_get_sequence *get_seq = data;
2069 : ktime_t now;
2070 : bool vblank_enabled;
2071 : int ret;
2072 :
2073 0 : if (!drm_core_check_feature(dev, DRIVER_MODESET))
2074 : return -EOPNOTSUPP;
2075 :
2076 0 : if (!drm_dev_has_vblank(dev))
2077 : return -EOPNOTSUPP;
2078 :
2079 0 : crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
2080 0 : if (!crtc)
2081 : return -ENOENT;
2082 :
2083 0 : pipe = drm_crtc_index(crtc);
2084 :
2085 0 : vblank = &dev->vblank[pipe];
2086 0 : vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
2087 :
2088 0 : if (!vblank_enabled) {
2089 0 : ret = drm_crtc_vblank_get(crtc);
2090 0 : if (ret) {
2091 0 : drm_dbg_core(dev,
2092 : "crtc %d failed to acquire vblank counter, %d\n",
2093 : pipe, ret);
2094 0 : return ret;
2095 : }
2096 : }
2097 0 : drm_modeset_lock(&crtc->mutex, NULL);
2098 0 : if (crtc->state)
2099 0 : get_seq->active = crtc->state->enable;
2100 : else
2101 0 : get_seq->active = crtc->enabled;
2102 0 : drm_modeset_unlock(&crtc->mutex);
2103 0 : get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
2104 0 : get_seq->sequence_ns = ktime_to_ns(now);
2105 0 : if (!vblank_enabled)
2106 : drm_crtc_vblank_put(crtc);
2107 : return 0;
2108 : }
2109 :
2110 : /*
2111 : * Queue a event for VBLANK sequence
2112 : *
2113 : * \param dev DRM device
2114 : * \param data user argument, pointing to a drm_crtc_queue_sequence structure.
2115 : * \param file_priv drm file private for the user's open file descriptor
2116 : */
2117 :
2118 0 : int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2119 : struct drm_file *file_priv)
2120 : {
2121 : struct drm_crtc *crtc;
2122 : struct drm_vblank_crtc *vblank;
2123 : int pipe;
2124 0 : struct drm_crtc_queue_sequence *queue_seq = data;
2125 : ktime_t now;
2126 : struct drm_pending_vblank_event *e;
2127 : u32 flags;
2128 : u64 seq;
2129 : u64 req_seq;
2130 : int ret;
2131 :
2132 0 : if (!drm_core_check_feature(dev, DRIVER_MODESET))
2133 : return -EOPNOTSUPP;
2134 :
2135 0 : if (!drm_dev_has_vblank(dev))
2136 : return -EOPNOTSUPP;
2137 :
2138 0 : crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
2139 0 : if (!crtc)
2140 : return -ENOENT;
2141 :
2142 0 : flags = queue_seq->flags;
2143 : /* Check valid flag bits */
2144 0 : if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2145 : DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2146 : return -EINVAL;
2147 :
2148 0 : pipe = drm_crtc_index(crtc);
2149 :
2150 0 : vblank = &dev->vblank[pipe];
2151 :
2152 0 : e = kzalloc(sizeof(*e), GFP_KERNEL);
2153 0 : if (e == NULL)
2154 : return -ENOMEM;
2155 :
2156 0 : ret = drm_crtc_vblank_get(crtc);
2157 0 : if (ret) {
2158 0 : drm_dbg_core(dev,
2159 : "crtc %d failed to acquire vblank counter, %d\n",
2160 : pipe, ret);
2161 0 : goto err_free;
2162 : }
2163 :
2164 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
2165 0 : req_seq = queue_seq->sequence;
2166 :
2167 0 : if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2168 0 : req_seq += seq;
2169 :
2170 0 : if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq))
2171 0 : req_seq = seq + 1;
2172 :
2173 0 : e->pipe = pipe;
2174 0 : e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2175 0 : e->event.base.length = sizeof(e->event.seq);
2176 0 : e->event.seq.user_data = queue_seq->user_data;
2177 :
2178 0 : spin_lock_irq(&dev->event_lock);
2179 :
2180 : /*
2181 : * drm_crtc_vblank_off() might have been called after we called
2182 : * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2183 : * vblank disable, so no need for further locking. The reference from
2184 : * drm_crtc_vblank_get() protects against vblank disable from another source.
2185 : */
2186 0 : if (!READ_ONCE(vblank->enabled)) {
2187 : ret = -EINVAL;
2188 : goto err_unlock;
2189 : }
2190 :
2191 0 : ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
2192 : &e->event.base);
2193 :
2194 0 : if (ret)
2195 : goto err_unlock;
2196 :
2197 0 : e->sequence = req_seq;
2198 :
2199 0 : if (drm_vblank_passed(seq, req_seq)) {
2200 0 : drm_crtc_vblank_put(crtc);
2201 0 : send_vblank_event(dev, e, seq, now);
2202 0 : queue_seq->sequence = seq;
2203 : } else {
2204 : /* drm_handle_vblank_events will call drm_vblank_put */
2205 0 : list_add_tail(&e->base.link, &dev->vblank_event_list);
2206 0 : queue_seq->sequence = req_seq;
2207 : }
2208 :
2209 0 : spin_unlock_irq(&dev->event_lock);
2210 0 : return 0;
2211 :
2212 : err_unlock:
2213 0 : spin_unlock_irq(&dev->event_lock);
2214 : drm_crtc_vblank_put(crtc);
2215 : err_free:
2216 0 : kfree(e);
2217 0 : return ret;
2218 : }
2219 :
|
