Line data Source code
1 : /*
2 : * Copyright 2017 Red Hat
3 : * Parts ported from amdgpu (fence wait code).
4 : * Copyright 2016 Advanced Micro Devices, Inc.
5 : *
6 : * Permission is hereby granted, free of charge, to any person obtaining a
7 : * copy of this software and associated documentation files (the "Software"),
8 : * to deal in the Software without restriction, including without limitation
9 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 : * and/or sell copies of the Software, and to permit persons to whom the
11 : * Software is furnished to do so, subject to the following conditions:
12 : *
13 : * The above copyright notice and this permission notice (including the next
14 : * paragraph) shall be included in all copies or substantial portions of the
15 : * Software.
16 : *
17 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23 : * IN THE SOFTWARE.
24 : *
25 : * Authors:
26 : *
27 : */
28 :
29 : /**
30 : * DOC: Overview
31 : *
32 : * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
33 : * container for a synchronization primitive which can be used by userspace
34 : * to explicitly synchronize GPU commands, can be shared between userspace
35 : * processes, and can be shared between different DRM drivers.
36 : * Their primary use-case is to implement Vulkan fences and semaphores.
37 : * The syncobj userspace API provides ioctls for several operations:
38 : *
39 : * - Creation and destruction of syncobjs
40 : * - Import and export of syncobjs to/from a syncobj file descriptor
41 : * - Import and export a syncobj's underlying fence to/from a sync file
42 : * - Reset a syncobj (set its fence to NULL)
43 : * - Signal a syncobj (set a trivially signaled fence)
44 : * - Wait for a syncobj's fence to appear and be signaled
45 : *
46 : * The syncobj userspace API also provides operations to manipulate a syncobj
47 : * in terms of a timeline of struct &dma_fence_chain rather than a single
48 : * struct &dma_fence, through the following operations:
49 : *
50 : * - Signal a given point on the timeline
51 : * - Wait for a given point to appear and/or be signaled
52 : * - Import and export from/to a given point of a timeline
53 : *
54 : * At it's core, a syncobj is simply a wrapper around a pointer to a struct
55 : * &dma_fence which may be NULL.
56 : * When a syncobj is first created, its pointer is either NULL or a pointer
57 : * to an already signaled fence depending on whether the
58 : * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
59 : * &DRM_IOCTL_SYNCOBJ_CREATE.
60 : *
61 : * If the syncobj is considered as a binary (its state is either signaled or
62 : * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
63 : * the syncobj, the syncobj's fence is replaced with a fence which will be
64 : * signaled by the completion of that work.
65 : * If the syncobj is considered as a timeline primitive, when GPU work is
66 : * enqueued in a DRM driver to signal the a given point of the syncobj, a new
67 : * struct &dma_fence_chain pointing to the DRM driver's fence and also
68 : * pointing to the previous fence that was in the syncobj. The new struct
69 : * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
70 : * completion of the DRM driver's work and also any work associated with the
71 : * fence previously in the syncobj.
72 : *
73 : * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
74 : * time the work is enqueued, it waits on the syncobj's fence before
75 : * submitting the work to hardware. That fence is either :
76 : *
77 : * - The syncobj's current fence if the syncobj is considered as a binary
78 : * primitive.
79 : * - The struct &dma_fence associated with a given point if the syncobj is
80 : * considered as a timeline primitive.
81 : *
82 : * If the syncobj's fence is NULL or not present in the syncobj's timeline,
83 : * the enqueue operation is expected to fail.
84 : *
85 : * With binary syncobj, all manipulation of the syncobjs's fence happens in
86 : * terms of the current fence at the time the ioctl is called by userspace
87 : * regardless of whether that operation is an immediate host-side operation
88 : * (signal or reset) or or an operation which is enqueued in some driver
89 : * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
90 : * to manipulate a syncobj from the host by resetting its pointer to NULL or
91 : * setting its pointer to a fence which is already signaled.
92 : *
93 : * With a timeline syncobj, all manipulation of the synobj's fence happens in
94 : * terms of a u64 value referring to point in the timeline. See
95 : * dma_fence_chain_find_seqno() to see how a given point is found in the
96 : * timeline.
97 : *
98 : * Note that applications should be careful to always use timeline set of
99 : * ioctl() when dealing with syncobj considered as timeline. Using a binary
100 : * set of ioctl() with a syncobj considered as timeline could result incorrect
101 : * synchronization. The use of binary syncobj is supported through the
102 : * timeline set of ioctl() by using a point value of 0, this will reproduce
103 : * the behavior of the binary set of ioctl() (for example replace the
104 : * syncobj's fence when signaling).
105 : *
106 : *
107 : * Host-side wait on syncobjs
108 : * --------------------------
109 : *
110 : * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
111 : * host-side wait on all of the syncobj fences simultaneously.
112 : * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
113 : * all of the syncobj fences to be signaled before it returns.
114 : * Otherwise, it returns once at least one syncobj fence has been signaled
115 : * and the index of a signaled fence is written back to the client.
116 : *
117 : * Unlike the enqueued GPU work dependencies which fail if they see a NULL
118 : * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
119 : * the host-side wait will first wait for the syncobj to receive a non-NULL
120 : * fence and then wait on that fence.
121 : * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
122 : * syncobjs in the array has a NULL fence, -EINVAL will be returned.
123 : * Assuming the syncobj starts off with a NULL fence, this allows a client
124 : * to do a host wait in one thread (or process) which waits on GPU work
125 : * submitted in another thread (or process) without having to manually
126 : * synchronize between the two.
127 : * This requirement is inherited from the Vulkan fence API.
128 : *
129 : * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
130 : * handles as well as an array of u64 points and does a host-side wait on all
131 : * of syncobj fences at the given points simultaneously.
132 : *
133 : * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
134 : * fence to materialize on the timeline without waiting for the fence to be
135 : * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
136 : * requirement is inherited from the wait-before-signal behavior required by
137 : * the Vulkan timeline semaphore API.
138 : *
139 : * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
140 : * blocking: an eventfd will be signaled when the syncobj is. This is useful to
141 : * integrate the wait in an event loop.
142 : *
143 : *
144 : * Import/export of syncobjs
145 : * -------------------------
146 : *
147 : * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
148 : * provide two mechanisms for import/export of syncobjs.
149 : *
150 : * The first lets the client import or export an entire syncobj to a file
151 : * descriptor.
152 : * These fd's are opaque and have no other use case, except passing the
153 : * syncobj between processes.
154 : * All exported file descriptors and any syncobj handles created as a
155 : * result of importing those file descriptors own a reference to the
156 : * same underlying struct &drm_syncobj and the syncobj can be used
157 : * persistently across all the processes with which it is shared.
158 : * The syncobj is freed only once the last reference is dropped.
159 : * Unlike dma-buf, importing a syncobj creates a new handle (with its own
160 : * reference) for every import instead of de-duplicating.
161 : * The primary use-case of this persistent import/export is for shared
162 : * Vulkan fences and semaphores.
163 : *
164 : * The second import/export mechanism, which is indicated by
165 : * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
166 : * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
167 : * import/export the syncobj's current fence from/to a &sync_file.
168 : * When a syncobj is exported to a sync file, that sync file wraps the
169 : * sycnobj's fence at the time of export and any later signal or reset
170 : * operations on the syncobj will not affect the exported sync file.
171 : * When a sync file is imported into a syncobj, the syncobj's fence is set
172 : * to the fence wrapped by that sync file.
173 : * Because sync files are immutable, resetting or signaling the syncobj
174 : * will not affect any sync files whose fences have been imported into the
175 : * syncobj.
176 : *
177 : *
178 : * Import/export of timeline points in timeline syncobjs
179 : * -----------------------------------------------------
180 : *
181 : * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
182 : * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
183 : * into another syncobj.
184 : *
185 : * Note that if you want to transfer a struct &dma_fence_chain from a given
186 : * point on a timeline syncobj from/into a binary syncobj, you can use the
187 : * point 0 to mean take/replace the fence in the syncobj.
188 : */
189 :
190 : #include <linux/anon_inodes.h>
191 : #include <linux/dma-fence-unwrap.h>
192 : #include <linux/eventfd.h>
193 : #include <linux/file.h>
194 : #include <linux/fs.h>
195 : #include <linux/sched/signal.h>
196 : #include <linux/sync_file.h>
197 : #include <linux/uaccess.h>
198 :
199 : #include <drm/drm.h>
200 : #include <drm/drm_drv.h>
201 : #include <drm/drm_file.h>
202 : #include <drm/drm_gem.h>
203 : #include <drm/drm_print.h>
204 : #include <drm/drm_syncobj.h>
205 : #include <drm/drm_utils.h>
206 :
207 : #include "drm_internal.h"
208 :
209 : struct syncobj_wait_entry {
210 : struct list_head node;
211 : struct task_struct *task;
212 : struct dma_fence *fence;
213 : struct dma_fence_cb fence_cb;
214 : u64 point;
215 : };
216 :
217 : static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
218 : struct syncobj_wait_entry *wait);
219 :
220 : struct syncobj_eventfd_entry {
221 : struct list_head node;
222 : struct dma_fence *fence;
223 : struct dma_fence_cb fence_cb;
224 : struct drm_syncobj *syncobj;
225 : struct eventfd_ctx *ev_fd_ctx;
226 : u64 point;
227 : u32 flags;
228 : };
229 :
230 : static void
231 : syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
232 : struct syncobj_eventfd_entry *entry);
233 :
234 : /**
235 : * drm_syncobj_find - lookup and reference a sync object.
236 : * @file_private: drm file private pointer
237 : * @handle: sync object handle to lookup.
238 : *
239 : * Returns a reference to the syncobj pointed to by handle or NULL. The
240 : * reference must be released by calling drm_syncobj_put().
241 : */
242 0 : struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
243 : u32 handle)
244 : {
245 : struct drm_syncobj *syncobj;
246 :
247 0 : spin_lock(&file_private->syncobj_table_lock);
248 :
249 : /* Check if we currently have a reference on the object */
250 0 : syncobj = idr_find(&file_private->syncobj_idr, handle);
251 0 : if (syncobj)
252 : drm_syncobj_get(syncobj);
253 :
254 0 : spin_unlock(&file_private->syncobj_table_lock);
255 :
256 0 : return syncobj;
257 : }
258 : EXPORT_SYMBOL(drm_syncobj_find);
259 :
260 0 : static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
261 : struct syncobj_wait_entry *wait)
262 : {
263 : struct dma_fence *fence;
264 :
265 0 : if (wait->fence)
266 0 : return;
267 :
268 0 : spin_lock(&syncobj->lock);
269 : /* We've already tried once to get a fence and failed. Now that we
270 : * have the lock, try one more time just to be sure we don't add a
271 : * callback when a fence has already been set.
272 : */
273 0 : fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
274 0 : if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
275 0 : dma_fence_put(fence);
276 0 : list_add_tail(&wait->node, &syncobj->cb_list);
277 0 : } else if (!fence) {
278 0 : wait->fence = dma_fence_get_stub();
279 : } else {
280 0 : wait->fence = fence;
281 : }
282 0 : spin_unlock(&syncobj->lock);
283 : }
284 :
285 : static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
286 : struct syncobj_wait_entry *wait)
287 : {
288 0 : if (!wait->node.next)
289 : return;
290 :
291 0 : spin_lock(&syncobj->lock);
292 0 : list_del_init(&wait->node);
293 0 : spin_unlock(&syncobj->lock);
294 : }
295 :
296 : static void
297 0 : syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
298 : {
299 0 : eventfd_ctx_put(entry->ev_fd_ctx);
300 0 : dma_fence_put(entry->fence);
301 : /* This happens either inside the syncobj lock, or after the node has
302 : * already been removed from the list.
303 : */
304 0 : list_del(&entry->node);
305 0 : kfree(entry);
306 0 : }
307 :
308 : static void
309 : drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
310 : struct syncobj_eventfd_entry *entry)
311 : {
312 0 : spin_lock(&syncobj->lock);
313 0 : list_add_tail(&entry->node, &syncobj->ev_fd_list);
314 0 : syncobj_eventfd_entry_func(syncobj, entry);
315 0 : spin_unlock(&syncobj->lock);
316 : }
317 :
318 : /**
319 : * drm_syncobj_add_point - add new timeline point to the syncobj
320 : * @syncobj: sync object to add timeline point do
321 : * @chain: chain node to use to add the point
322 : * @fence: fence to encapsulate in the chain node
323 : * @point: sequence number to use for the point
324 : *
325 : * Add the chain node as new timeline point to the syncobj.
326 : */
327 0 : void drm_syncobj_add_point(struct drm_syncobj *syncobj,
328 : struct dma_fence_chain *chain,
329 : struct dma_fence *fence,
330 : uint64_t point)
331 : {
332 : struct syncobj_wait_entry *wait_cur, *wait_tmp;
333 : struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
334 : struct dma_fence *prev;
335 :
336 0 : dma_fence_get(fence);
337 :
338 0 : spin_lock(&syncobj->lock);
339 :
340 0 : prev = drm_syncobj_fence_get(syncobj);
341 : /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
342 0 : if (prev && prev->seqno >= point)
343 0 : DRM_DEBUG("You are adding an unorder point to timeline!\n");
344 0 : dma_fence_chain_init(chain, prev, fence, point);
345 0 : rcu_assign_pointer(syncobj->fence, &chain->base);
346 :
347 0 : list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
348 0 : syncobj_wait_syncobj_func(syncobj, wait_cur);
349 0 : list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
350 0 : syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
351 0 : spin_unlock(&syncobj->lock);
352 :
353 : /* Walk the chain once to trigger garbage collection */
354 0 : dma_fence_chain_for_each(fence, prev);
355 0 : dma_fence_put(prev);
356 0 : }
357 : EXPORT_SYMBOL(drm_syncobj_add_point);
358 :
359 : /**
360 : * drm_syncobj_replace_fence - replace fence in a sync object.
361 : * @syncobj: Sync object to replace fence in
362 : * @fence: fence to install in sync file.
363 : *
364 : * This replaces the fence on a sync object.
365 : */
366 0 : void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
367 : struct dma_fence *fence)
368 : {
369 : struct dma_fence *old_fence;
370 : struct syncobj_wait_entry *wait_cur, *wait_tmp;
371 : struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
372 :
373 0 : if (fence)
374 : dma_fence_get(fence);
375 :
376 0 : spin_lock(&syncobj->lock);
377 :
378 0 : old_fence = rcu_dereference_protected(syncobj->fence,
379 : lockdep_is_held(&syncobj->lock));
380 0 : rcu_assign_pointer(syncobj->fence, fence);
381 :
382 0 : if (fence != old_fence) {
383 0 : list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
384 0 : syncobj_wait_syncobj_func(syncobj, wait_cur);
385 0 : list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
386 0 : syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
387 : }
388 :
389 0 : spin_unlock(&syncobj->lock);
390 :
391 0 : dma_fence_put(old_fence);
392 0 : }
393 : EXPORT_SYMBOL(drm_syncobj_replace_fence);
394 :
395 : /**
396 : * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
397 : * @syncobj: sync object to assign the fence on
398 : *
399 : * Assign a already signaled stub fence to the sync object.
400 : */
401 0 : static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
402 : {
403 0 : struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());
404 :
405 0 : if (!fence)
406 : return -ENOMEM;
407 :
408 0 : drm_syncobj_replace_fence(syncobj, fence);
409 0 : dma_fence_put(fence);
410 0 : return 0;
411 : }
412 :
413 : /* 5s default for wait submission */
414 : #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
415 : /**
416 : * drm_syncobj_find_fence - lookup and reference the fence in a sync object
417 : * @file_private: drm file private pointer
418 : * @handle: sync object handle to lookup.
419 : * @point: timeline point
420 : * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
421 : * @fence: out parameter for the fence
422 : *
423 : * This is just a convenience function that combines drm_syncobj_find() and
424 : * drm_syncobj_fence_get().
425 : *
426 : * Returns 0 on success or a negative error value on failure. On success @fence
427 : * contains a reference to the fence, which must be released by calling
428 : * dma_fence_put().
429 : */
430 0 : int drm_syncobj_find_fence(struct drm_file *file_private,
431 : u32 handle, u64 point, u64 flags,
432 : struct dma_fence **fence)
433 : {
434 0 : struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
435 : struct syncobj_wait_entry wait;
436 0 : u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
437 : int ret;
438 :
439 0 : if (!syncobj)
440 : return -ENOENT;
441 :
442 : /* Waiting for userspace with locks help is illegal cause that can
443 : * trivial deadlock with page faults for example. Make lockdep complain
444 : * about it early on.
445 : */
446 0 : if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
447 : might_sleep();
448 : lockdep_assert_none_held_once();
449 : }
450 :
451 0 : *fence = drm_syncobj_fence_get(syncobj);
452 :
453 0 : if (*fence) {
454 0 : ret = dma_fence_chain_find_seqno(fence, point);
455 0 : if (!ret) {
456 : /* If the requested seqno is already signaled
457 : * drm_syncobj_find_fence may return a NULL
458 : * fence. To make sure the recipient gets
459 : * signalled, use a new fence instead.
460 : */
461 0 : if (!*fence)
462 0 : *fence = dma_fence_get_stub();
463 :
464 : goto out;
465 : }
466 0 : dma_fence_put(*fence);
467 : } else {
468 : ret = -EINVAL;
469 : }
470 :
471 0 : if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
472 : goto out;
473 :
474 0 : memset(&wait, 0, sizeof(wait));
475 0 : wait.task = current;
476 0 : wait.point = point;
477 0 : drm_syncobj_fence_add_wait(syncobj, &wait);
478 :
479 : do {
480 0 : set_current_state(TASK_INTERRUPTIBLE);
481 0 : if (wait.fence) {
482 : ret = 0;
483 : break;
484 : }
485 0 : if (timeout == 0) {
486 : ret = -ETIME;
487 : break;
488 : }
489 :
490 0 : if (signal_pending(current)) {
491 : ret = -ERESTARTSYS;
492 : break;
493 : }
494 :
495 0 : timeout = schedule_timeout(timeout);
496 : } while (1);
497 :
498 0 : __set_current_state(TASK_RUNNING);
499 0 : *fence = wait.fence;
500 :
501 0 : if (wait.node.next)
502 0 : drm_syncobj_remove_wait(syncobj, &wait);
503 :
504 : out:
505 0 : drm_syncobj_put(syncobj);
506 :
507 0 : return ret;
508 : }
509 : EXPORT_SYMBOL(drm_syncobj_find_fence);
510 :
511 : /**
512 : * drm_syncobj_free - free a sync object.
513 : * @kref: kref to free.
514 : *
515 : * Only to be called from kref_put in drm_syncobj_put.
516 : */
517 0 : void drm_syncobj_free(struct kref *kref)
518 : {
519 0 : struct drm_syncobj *syncobj = container_of(kref,
520 : struct drm_syncobj,
521 : refcount);
522 : struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
523 :
524 0 : drm_syncobj_replace_fence(syncobj, NULL);
525 :
526 0 : list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
527 0 : syncobj_eventfd_entry_free(ev_fd_cur);
528 :
529 0 : kfree(syncobj);
530 0 : }
531 : EXPORT_SYMBOL(drm_syncobj_free);
532 :
533 : /**
534 : * drm_syncobj_create - create a new syncobj
535 : * @out_syncobj: returned syncobj
536 : * @flags: DRM_SYNCOBJ_* flags
537 : * @fence: if non-NULL, the syncobj will represent this fence
538 : *
539 : * This is the first function to create a sync object. After creating, drivers
540 : * probably want to make it available to userspace, either through
541 : * drm_syncobj_get_handle() or drm_syncobj_get_fd().
542 : *
543 : * Returns 0 on success or a negative error value on failure.
544 : */
545 0 : int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
546 : struct dma_fence *fence)
547 : {
548 : int ret;
549 : struct drm_syncobj *syncobj;
550 :
551 0 : syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
552 0 : if (!syncobj)
553 : return -ENOMEM;
554 :
555 0 : kref_init(&syncobj->refcount);
556 0 : INIT_LIST_HEAD(&syncobj->cb_list);
557 0 : INIT_LIST_HEAD(&syncobj->ev_fd_list);
558 0 : spin_lock_init(&syncobj->lock);
559 :
560 0 : if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
561 0 : ret = drm_syncobj_assign_null_handle(syncobj);
562 0 : if (ret < 0) {
563 0 : drm_syncobj_put(syncobj);
564 0 : return ret;
565 : }
566 : }
567 :
568 0 : if (fence)
569 0 : drm_syncobj_replace_fence(syncobj, fence);
570 :
571 0 : *out_syncobj = syncobj;
572 0 : return 0;
573 : }
574 : EXPORT_SYMBOL(drm_syncobj_create);
575 :
576 : /**
577 : * drm_syncobj_get_handle - get a handle from a syncobj
578 : * @file_private: drm file private pointer
579 : * @syncobj: Sync object to export
580 : * @handle: out parameter with the new handle
581 : *
582 : * Exports a sync object created with drm_syncobj_create() as a handle on
583 : * @file_private to userspace.
584 : *
585 : * Returns 0 on success or a negative error value on failure.
586 : */
587 0 : int drm_syncobj_get_handle(struct drm_file *file_private,
588 : struct drm_syncobj *syncobj, u32 *handle)
589 : {
590 : int ret;
591 :
592 : /* take a reference to put in the idr */
593 0 : drm_syncobj_get(syncobj);
594 :
595 0 : idr_preload(GFP_KERNEL);
596 0 : spin_lock(&file_private->syncobj_table_lock);
597 0 : ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
598 0 : spin_unlock(&file_private->syncobj_table_lock);
599 :
600 : idr_preload_end();
601 :
602 0 : if (ret < 0) {
603 0 : drm_syncobj_put(syncobj);
604 0 : return ret;
605 : }
606 :
607 0 : *handle = ret;
608 0 : return 0;
609 : }
610 : EXPORT_SYMBOL(drm_syncobj_get_handle);
611 :
612 0 : static int drm_syncobj_create_as_handle(struct drm_file *file_private,
613 : u32 *handle, uint32_t flags)
614 : {
615 : int ret;
616 : struct drm_syncobj *syncobj;
617 :
618 0 : ret = drm_syncobj_create(&syncobj, flags, NULL);
619 0 : if (ret)
620 : return ret;
621 :
622 0 : ret = drm_syncobj_get_handle(file_private, syncobj, handle);
623 0 : drm_syncobj_put(syncobj);
624 0 : return ret;
625 : }
626 :
627 0 : static int drm_syncobj_destroy(struct drm_file *file_private,
628 : u32 handle)
629 : {
630 : struct drm_syncobj *syncobj;
631 :
632 0 : spin_lock(&file_private->syncobj_table_lock);
633 0 : syncobj = idr_remove(&file_private->syncobj_idr, handle);
634 0 : spin_unlock(&file_private->syncobj_table_lock);
635 :
636 0 : if (!syncobj)
637 : return -EINVAL;
638 :
639 0 : drm_syncobj_put(syncobj);
640 0 : return 0;
641 : }
642 :
643 0 : static int drm_syncobj_file_release(struct inode *inode, struct file *file)
644 : {
645 0 : struct drm_syncobj *syncobj = file->private_data;
646 :
647 0 : drm_syncobj_put(syncobj);
648 0 : return 0;
649 : }
650 :
651 : static const struct file_operations drm_syncobj_file_fops = {
652 : .release = drm_syncobj_file_release,
653 : };
654 :
655 : /**
656 : * drm_syncobj_get_fd - get a file descriptor from a syncobj
657 : * @syncobj: Sync object to export
658 : * @p_fd: out parameter with the new file descriptor
659 : *
660 : * Exports a sync object created with drm_syncobj_create() as a file descriptor.
661 : *
662 : * Returns 0 on success or a negative error value on failure.
663 : */
664 0 : int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
665 : {
666 : struct file *file;
667 : int fd;
668 :
669 0 : fd = get_unused_fd_flags(O_CLOEXEC);
670 0 : if (fd < 0)
671 : return fd;
672 :
673 0 : file = anon_inode_getfile("syncobj_file",
674 : &drm_syncobj_file_fops,
675 : syncobj, 0);
676 0 : if (IS_ERR(file)) {
677 0 : put_unused_fd(fd);
678 0 : return PTR_ERR(file);
679 : }
680 :
681 0 : drm_syncobj_get(syncobj);
682 0 : fd_install(fd, file);
683 :
684 0 : *p_fd = fd;
685 0 : return 0;
686 : }
687 : EXPORT_SYMBOL(drm_syncobj_get_fd);
688 :
689 0 : static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
690 : u32 handle, int *p_fd)
691 : {
692 0 : struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
693 : int ret;
694 :
695 0 : if (!syncobj)
696 : return -EINVAL;
697 :
698 0 : ret = drm_syncobj_get_fd(syncobj, p_fd);
699 0 : drm_syncobj_put(syncobj);
700 0 : return ret;
701 : }
702 :
703 0 : static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
704 : int fd, u32 *handle)
705 : {
706 : struct drm_syncobj *syncobj;
707 0 : struct fd f = fdget(fd);
708 : int ret;
709 :
710 0 : if (!f.file)
711 : return -EINVAL;
712 :
713 0 : if (f.file->f_op != &drm_syncobj_file_fops) {
714 0 : fdput(f);
715 : return -EINVAL;
716 : }
717 :
718 : /* take a reference to put in the idr */
719 0 : syncobj = f.file->private_data;
720 0 : drm_syncobj_get(syncobj);
721 :
722 0 : idr_preload(GFP_KERNEL);
723 0 : spin_lock(&file_private->syncobj_table_lock);
724 0 : ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
725 0 : spin_unlock(&file_private->syncobj_table_lock);
726 : idr_preload_end();
727 :
728 0 : if (ret > 0) {
729 0 : *handle = ret;
730 0 : ret = 0;
731 : } else
732 : drm_syncobj_put(syncobj);
733 :
734 0 : fdput(f);
735 : return ret;
736 : }
737 :
738 0 : static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
739 : int fd, int handle)
740 : {
741 0 : struct dma_fence *fence = sync_file_get_fence(fd);
742 : struct drm_syncobj *syncobj;
743 :
744 0 : if (!fence)
745 : return -EINVAL;
746 :
747 0 : syncobj = drm_syncobj_find(file_private, handle);
748 0 : if (!syncobj) {
749 0 : dma_fence_put(fence);
750 0 : return -ENOENT;
751 : }
752 :
753 0 : drm_syncobj_replace_fence(syncobj, fence);
754 0 : dma_fence_put(fence);
755 0 : drm_syncobj_put(syncobj);
756 0 : return 0;
757 : }
758 :
759 0 : static int drm_syncobj_export_sync_file(struct drm_file *file_private,
760 : int handle, int *p_fd)
761 : {
762 : int ret;
763 : struct dma_fence *fence;
764 : struct sync_file *sync_file;
765 0 : int fd = get_unused_fd_flags(O_CLOEXEC);
766 :
767 0 : if (fd < 0)
768 : return fd;
769 :
770 0 : ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
771 0 : if (ret)
772 : goto err_put_fd;
773 :
774 0 : sync_file = sync_file_create(fence);
775 :
776 0 : dma_fence_put(fence);
777 :
778 0 : if (!sync_file) {
779 : ret = -EINVAL;
780 : goto err_put_fd;
781 : }
782 :
783 0 : fd_install(fd, sync_file->file);
784 :
785 0 : *p_fd = fd;
786 0 : return 0;
787 : err_put_fd:
788 0 : put_unused_fd(fd);
789 0 : return ret;
790 : }
791 : /**
792 : * drm_syncobj_open - initializes syncobj file-private structures at devnode open time
793 : * @file_private: drm file-private structure to set up
794 : *
795 : * Called at device open time, sets up the structure for handling refcounting
796 : * of sync objects.
797 : */
798 : void
799 0 : drm_syncobj_open(struct drm_file *file_private)
800 : {
801 0 : idr_init_base(&file_private->syncobj_idr, 1);
802 0 : spin_lock_init(&file_private->syncobj_table_lock);
803 0 : }
804 :
805 : static int
806 0 : drm_syncobj_release_handle(int id, void *ptr, void *data)
807 : {
808 0 : struct drm_syncobj *syncobj = ptr;
809 :
810 0 : drm_syncobj_put(syncobj);
811 0 : return 0;
812 : }
813 :
814 : /**
815 : * drm_syncobj_release - release file-private sync object resources
816 : * @file_private: drm file-private structure to clean up
817 : *
818 : * Called at close time when the filp is going away.
819 : *
820 : * Releases any remaining references on objects by this filp.
821 : */
822 : void
823 0 : drm_syncobj_release(struct drm_file *file_private)
824 : {
825 0 : idr_for_each(&file_private->syncobj_idr,
826 : &drm_syncobj_release_handle, file_private);
827 0 : idr_destroy(&file_private->syncobj_idr);
828 0 : }
829 :
830 : int
831 0 : drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
832 : struct drm_file *file_private)
833 : {
834 0 : struct drm_syncobj_create *args = data;
835 :
836 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
837 : return -EOPNOTSUPP;
838 :
839 : /* no valid flags yet */
840 0 : if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
841 : return -EINVAL;
842 :
843 0 : return drm_syncobj_create_as_handle(file_private,
844 0 : &args->handle, args->flags);
845 : }
846 :
847 : int
848 0 : drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
849 : struct drm_file *file_private)
850 : {
851 0 : struct drm_syncobj_destroy *args = data;
852 :
853 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
854 : return -EOPNOTSUPP;
855 :
856 : /* make sure padding is empty */
857 0 : if (args->pad)
858 : return -EINVAL;
859 0 : return drm_syncobj_destroy(file_private, args->handle);
860 : }
861 :
862 : int
863 0 : drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
864 : struct drm_file *file_private)
865 : {
866 0 : struct drm_syncobj_handle *args = data;
867 :
868 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
869 : return -EOPNOTSUPP;
870 :
871 0 : if (args->pad)
872 : return -EINVAL;
873 :
874 0 : if (args->flags != 0 &&
875 : args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
876 : return -EINVAL;
877 :
878 0 : if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
879 0 : return drm_syncobj_export_sync_file(file_private, args->handle,
880 0 : &args->fd);
881 :
882 0 : return drm_syncobj_handle_to_fd(file_private, args->handle,
883 0 : &args->fd);
884 : }
885 :
886 : int
887 0 : drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
888 : struct drm_file *file_private)
889 : {
890 0 : struct drm_syncobj_handle *args = data;
891 :
892 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
893 : return -EOPNOTSUPP;
894 :
895 0 : if (args->pad)
896 : return -EINVAL;
897 :
898 0 : if (args->flags != 0 &&
899 : args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
900 : return -EINVAL;
901 :
902 0 : if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
903 0 : return drm_syncobj_import_sync_file_fence(file_private,
904 : args->fd,
905 0 : args->handle);
906 :
907 0 : return drm_syncobj_fd_to_handle(file_private, args->fd,
908 0 : &args->handle);
909 : }
910 :
911 0 : static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
912 : struct drm_syncobj_transfer *args)
913 : {
914 0 : struct drm_syncobj *timeline_syncobj = NULL;
915 : struct dma_fence *fence, *tmp;
916 : struct dma_fence_chain *chain;
917 : int ret;
918 :
919 0 : timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
920 0 : if (!timeline_syncobj) {
921 : return -ENOENT;
922 : }
923 0 : ret = drm_syncobj_find_fence(file_private, args->src_handle,
924 0 : args->src_point, args->flags,
925 : &tmp);
926 0 : if (ret)
927 : goto err_put_timeline;
928 :
929 0 : fence = dma_fence_unwrap_merge(tmp);
930 0 : dma_fence_put(tmp);
931 0 : if (!fence) {
932 : ret = -ENOMEM;
933 : goto err_put_timeline;
934 : }
935 :
936 0 : chain = dma_fence_chain_alloc();
937 0 : if (!chain) {
938 : ret = -ENOMEM;
939 : goto err_free_fence;
940 : }
941 :
942 0 : drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
943 : err_free_fence:
944 : dma_fence_put(fence);
945 : err_put_timeline:
946 0 : drm_syncobj_put(timeline_syncobj);
947 :
948 0 : return ret;
949 : }
950 :
951 : static int
952 0 : drm_syncobj_transfer_to_binary(struct drm_file *file_private,
953 : struct drm_syncobj_transfer *args)
954 : {
955 0 : struct drm_syncobj *binary_syncobj = NULL;
956 : struct dma_fence *fence;
957 : int ret;
958 :
959 0 : binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
960 0 : if (!binary_syncobj)
961 : return -ENOENT;
962 0 : ret = drm_syncobj_find_fence(file_private, args->src_handle,
963 0 : args->src_point, args->flags, &fence);
964 0 : if (ret)
965 : goto err;
966 0 : drm_syncobj_replace_fence(binary_syncobj, fence);
967 0 : dma_fence_put(fence);
968 : err:
969 0 : drm_syncobj_put(binary_syncobj);
970 :
971 0 : return ret;
972 : }
973 : int
974 0 : drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
975 : struct drm_file *file_private)
976 : {
977 0 : struct drm_syncobj_transfer *args = data;
978 : int ret;
979 :
980 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
981 : return -EOPNOTSUPP;
982 :
983 0 : if (args->pad)
984 : return -EINVAL;
985 :
986 0 : if (args->dst_point)
987 0 : ret = drm_syncobj_transfer_to_timeline(file_private, args);
988 : else
989 0 : ret = drm_syncobj_transfer_to_binary(file_private, args);
990 :
991 : return ret;
992 : }
993 :
994 0 : static void syncobj_wait_fence_func(struct dma_fence *fence,
995 : struct dma_fence_cb *cb)
996 : {
997 0 : struct syncobj_wait_entry *wait =
998 0 : container_of(cb, struct syncobj_wait_entry, fence_cb);
999 :
1000 0 : wake_up_process(wait->task);
1001 0 : }
1002 :
1003 0 : static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1004 : struct syncobj_wait_entry *wait)
1005 : {
1006 : struct dma_fence *fence;
1007 :
1008 : /* This happens inside the syncobj lock */
1009 0 : fence = rcu_dereference_protected(syncobj->fence,
1010 : lockdep_is_held(&syncobj->lock));
1011 0 : dma_fence_get(fence);
1012 0 : if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
1013 0 : dma_fence_put(fence);
1014 0 : return;
1015 0 : } else if (!fence) {
1016 0 : wait->fence = dma_fence_get_stub();
1017 : } else {
1018 0 : wait->fence = fence;
1019 : }
1020 :
1021 0 : wake_up_process(wait->task);
1022 0 : list_del_init(&wait->node);
1023 : }
1024 :
1025 0 : static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1026 : void __user *user_points,
1027 : uint32_t count,
1028 : uint32_t flags,
1029 : signed long timeout,
1030 : uint32_t *idx)
1031 : {
1032 : struct syncobj_wait_entry *entries;
1033 : struct dma_fence *fence;
1034 : uint64_t *points;
1035 : uint32_t signaled_count, i;
1036 :
1037 0 : if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
1038 : lockdep_assert_none_held_once();
1039 :
1040 0 : points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
1041 0 : if (points == NULL)
1042 : return -ENOMEM;
1043 :
1044 0 : if (!user_points) {
1045 0 : memset(points, 0, count * sizeof(uint64_t));
1046 :
1047 0 : } else if (copy_from_user(points, user_points,
1048 : sizeof(uint64_t) * count)) {
1049 : timeout = -EFAULT;
1050 : goto err_free_points;
1051 : }
1052 :
1053 0 : entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
1054 0 : if (!entries) {
1055 : timeout = -ENOMEM;
1056 : goto err_free_points;
1057 : }
1058 : /* Walk the list of sync objects and initialize entries. We do
1059 : * this up-front so that we can properly return -EINVAL if there is
1060 : * a syncobj with a missing fence and then never have the chance of
1061 : * returning -EINVAL again.
1062 : */
1063 : signaled_count = 0;
1064 0 : for (i = 0; i < count; ++i) {
1065 : struct dma_fence *fence;
1066 :
1067 0 : entries[i].task = current;
1068 0 : entries[i].point = points[i];
1069 0 : fence = drm_syncobj_fence_get(syncobjs[i]);
1070 0 : if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
1071 0 : dma_fence_put(fence);
1072 0 : if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1073 0 : continue;
1074 : } else {
1075 0 : timeout = -EINVAL;
1076 0 : goto cleanup_entries;
1077 : }
1078 : }
1079 :
1080 0 : if (fence)
1081 0 : entries[i].fence = fence;
1082 : else
1083 0 : entries[i].fence = dma_fence_get_stub();
1084 :
1085 0 : if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1086 0 : dma_fence_is_signaled(entries[i].fence)) {
1087 0 : if (signaled_count == 0 && idx)
1088 0 : *idx = i;
1089 0 : signaled_count++;
1090 : }
1091 : }
1092 :
1093 0 : if (signaled_count == count ||
1094 0 : (signaled_count > 0 &&
1095 0 : !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1096 : goto cleanup_entries;
1097 :
1098 : /* There's a very annoying laxness in the dma_fence API here, in
1099 : * that backends are not required to automatically report when a
1100 : * fence is signaled prior to fence->ops->enable_signaling() being
1101 : * called. So here if we fail to match signaled_count, we need to
1102 : * fallthough and try a 0 timeout wait!
1103 : */
1104 :
1105 0 : if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1106 0 : for (i = 0; i < count; ++i)
1107 0 : drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1108 : }
1109 :
1110 : do {
1111 0 : set_current_state(TASK_INTERRUPTIBLE);
1112 :
1113 0 : signaled_count = 0;
1114 0 : for (i = 0; i < count; ++i) {
1115 0 : fence = entries[i].fence;
1116 0 : if (!fence)
1117 0 : continue;
1118 :
1119 0 : if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1120 0 : dma_fence_is_signaled(fence) ||
1121 0 : (!entries[i].fence_cb.func &&
1122 0 : dma_fence_add_callback(fence,
1123 : &entries[i].fence_cb,
1124 : syncobj_wait_fence_func))) {
1125 : /* The fence has been signaled */
1126 0 : if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1127 0 : signaled_count++;
1128 : } else {
1129 0 : if (idx)
1130 0 : *idx = i;
1131 : goto done_waiting;
1132 : }
1133 : }
1134 : }
1135 :
1136 0 : if (signaled_count == count)
1137 : goto done_waiting;
1138 :
1139 0 : if (timeout == 0) {
1140 : timeout = -ETIME;
1141 : goto done_waiting;
1142 : }
1143 :
1144 0 : if (signal_pending(current)) {
1145 : timeout = -ERESTARTSYS;
1146 : goto done_waiting;
1147 : }
1148 :
1149 0 : timeout = schedule_timeout(timeout);
1150 : } while (1);
1151 :
1152 : done_waiting:
1153 0 : __set_current_state(TASK_RUNNING);
1154 :
1155 : cleanup_entries:
1156 0 : for (i = 0; i < count; ++i) {
1157 0 : drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1158 0 : if (entries[i].fence_cb.func)
1159 0 : dma_fence_remove_callback(entries[i].fence,
1160 : &entries[i].fence_cb);
1161 0 : dma_fence_put(entries[i].fence);
1162 : }
1163 0 : kfree(entries);
1164 :
1165 : err_free_points:
1166 0 : kfree(points);
1167 :
1168 0 : return timeout;
1169 : }
1170 :
1171 : /**
1172 : * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1173 : *
1174 : * @timeout_nsec: timeout nsec component in ns, 0 for poll
1175 : *
1176 : * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1177 : */
1178 0 : signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1179 : {
1180 : ktime_t abs_timeout, now;
1181 : u64 timeout_ns, timeout_jiffies64;
1182 :
1183 : /* make 0 timeout means poll - absolute 0 doesn't seem valid */
1184 0 : if (timeout_nsec == 0)
1185 : return 0;
1186 :
1187 0 : abs_timeout = ns_to_ktime(timeout_nsec);
1188 0 : now = ktime_get();
1189 :
1190 0 : if (!ktime_after(abs_timeout, now))
1191 : return 0;
1192 :
1193 0 : timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1194 :
1195 0 : timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1196 : /* clamp timeout to avoid infinite timeout */
1197 0 : if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1198 : return MAX_SCHEDULE_TIMEOUT - 1;
1199 :
1200 0 : return timeout_jiffies64 + 1;
1201 : }
1202 : EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1203 :
1204 0 : static int drm_syncobj_array_wait(struct drm_device *dev,
1205 : struct drm_file *file_private,
1206 : struct drm_syncobj_wait *wait,
1207 : struct drm_syncobj_timeline_wait *timeline_wait,
1208 : struct drm_syncobj **syncobjs, bool timeline)
1209 : {
1210 0 : signed long timeout = 0;
1211 0 : uint32_t first = ~0;
1212 :
1213 0 : if (!timeline) {
1214 0 : timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1215 0 : timeout = drm_syncobj_array_wait_timeout(syncobjs,
1216 : NULL,
1217 : wait->count_handles,
1218 : wait->flags,
1219 : timeout, &first);
1220 0 : if (timeout < 0)
1221 0 : return timeout;
1222 0 : wait->first_signaled = first;
1223 : } else {
1224 0 : timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1225 0 : timeout = drm_syncobj_array_wait_timeout(syncobjs,
1226 0 : u64_to_user_ptr(timeline_wait->points),
1227 : timeline_wait->count_handles,
1228 : timeline_wait->flags,
1229 : timeout, &first);
1230 0 : if (timeout < 0)
1231 0 : return timeout;
1232 0 : timeline_wait->first_signaled = first;
1233 : }
1234 : return 0;
1235 : }
1236 :
1237 0 : static int drm_syncobj_array_find(struct drm_file *file_private,
1238 : void __user *user_handles,
1239 : uint32_t count_handles,
1240 : struct drm_syncobj ***syncobjs_out)
1241 : {
1242 : uint32_t i, *handles;
1243 : struct drm_syncobj **syncobjs;
1244 : int ret;
1245 :
1246 0 : handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1247 0 : if (handles == NULL)
1248 : return -ENOMEM;
1249 :
1250 0 : if (copy_from_user(handles, user_handles,
1251 : sizeof(uint32_t) * count_handles)) {
1252 : ret = -EFAULT;
1253 : goto err_free_handles;
1254 : }
1255 :
1256 0 : syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1257 0 : if (syncobjs == NULL) {
1258 : ret = -ENOMEM;
1259 : goto err_free_handles;
1260 : }
1261 :
1262 0 : for (i = 0; i < count_handles; i++) {
1263 0 : syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1264 0 : if (!syncobjs[i]) {
1265 : ret = -ENOENT;
1266 : goto err_put_syncobjs;
1267 : }
1268 : }
1269 :
1270 0 : kfree(handles);
1271 0 : *syncobjs_out = syncobjs;
1272 0 : return 0;
1273 :
1274 : err_put_syncobjs:
1275 0 : while (i-- > 0)
1276 0 : drm_syncobj_put(syncobjs[i]);
1277 0 : kfree(syncobjs);
1278 : err_free_handles:
1279 0 : kfree(handles);
1280 :
1281 0 : return ret;
1282 : }
1283 :
1284 0 : static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1285 : uint32_t count)
1286 : {
1287 : uint32_t i;
1288 :
1289 0 : for (i = 0; i < count; i++)
1290 0 : drm_syncobj_put(syncobjs[i]);
1291 0 : kfree(syncobjs);
1292 0 : }
1293 :
1294 : int
1295 0 : drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1296 : struct drm_file *file_private)
1297 : {
1298 0 : struct drm_syncobj_wait *args = data;
1299 : struct drm_syncobj **syncobjs;
1300 0 : int ret = 0;
1301 :
1302 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1303 : return -EOPNOTSUPP;
1304 :
1305 0 : if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1306 : DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1307 : return -EINVAL;
1308 :
1309 0 : if (args->count_handles == 0)
1310 : return -EINVAL;
1311 :
1312 0 : ret = drm_syncobj_array_find(file_private,
1313 0 : u64_to_user_ptr(args->handles),
1314 : args->count_handles,
1315 : &syncobjs);
1316 0 : if (ret < 0)
1317 : return ret;
1318 :
1319 0 : ret = drm_syncobj_array_wait(dev, file_private,
1320 : args, NULL, syncobjs, false);
1321 :
1322 0 : drm_syncobj_array_free(syncobjs, args->count_handles);
1323 :
1324 0 : return ret;
1325 : }
1326 :
1327 : int
1328 0 : drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1329 : struct drm_file *file_private)
1330 : {
1331 0 : struct drm_syncobj_timeline_wait *args = data;
1332 : struct drm_syncobj **syncobjs;
1333 0 : int ret = 0;
1334 :
1335 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1336 : return -EOPNOTSUPP;
1337 :
1338 0 : if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1339 : DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1340 : DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1341 : return -EINVAL;
1342 :
1343 0 : if (args->count_handles == 0)
1344 : return -EINVAL;
1345 :
1346 0 : ret = drm_syncobj_array_find(file_private,
1347 0 : u64_to_user_ptr(args->handles),
1348 : args->count_handles,
1349 : &syncobjs);
1350 0 : if (ret < 0)
1351 : return ret;
1352 :
1353 0 : ret = drm_syncobj_array_wait(dev, file_private,
1354 : NULL, args, syncobjs, true);
1355 :
1356 0 : drm_syncobj_array_free(syncobjs, args->count_handles);
1357 :
1358 0 : return ret;
1359 : }
1360 :
1361 0 : static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1362 : struct dma_fence_cb *cb)
1363 : {
1364 0 : struct syncobj_eventfd_entry *entry =
1365 0 : container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1366 :
1367 0 : eventfd_signal(entry->ev_fd_ctx, 1);
1368 0 : syncobj_eventfd_entry_free(entry);
1369 0 : }
1370 :
1371 : static void
1372 0 : syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1373 : struct syncobj_eventfd_entry *entry)
1374 : {
1375 : int ret;
1376 : struct dma_fence *fence;
1377 :
1378 : /* This happens inside the syncobj lock */
1379 0 : fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1380 0 : ret = dma_fence_chain_find_seqno(&fence, entry->point);
1381 0 : if (ret != 0 || !fence) {
1382 0 : dma_fence_put(fence);
1383 0 : return;
1384 : }
1385 :
1386 0 : list_del_init(&entry->node);
1387 0 : entry->fence = fence;
1388 :
1389 0 : if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1390 0 : eventfd_signal(entry->ev_fd_ctx, 1);
1391 0 : syncobj_eventfd_entry_free(entry);
1392 : } else {
1393 0 : ret = dma_fence_add_callback(fence, &entry->fence_cb,
1394 : syncobj_eventfd_entry_fence_func);
1395 0 : if (ret == -ENOENT) {
1396 0 : eventfd_signal(entry->ev_fd_ctx, 1);
1397 0 : syncobj_eventfd_entry_free(entry);
1398 : }
1399 : }
1400 : }
1401 :
1402 : int
1403 0 : drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1404 : struct drm_file *file_private)
1405 : {
1406 0 : struct drm_syncobj_eventfd *args = data;
1407 : struct drm_syncobj *syncobj;
1408 : struct eventfd_ctx *ev_fd_ctx;
1409 : struct syncobj_eventfd_entry *entry;
1410 :
1411 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1412 : return -EOPNOTSUPP;
1413 :
1414 0 : if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1415 : return -EINVAL;
1416 :
1417 0 : if (args->pad)
1418 : return -EINVAL;
1419 :
1420 0 : syncobj = drm_syncobj_find(file_private, args->handle);
1421 0 : if (!syncobj)
1422 : return -ENOENT;
1423 :
1424 0 : ev_fd_ctx = eventfd_ctx_fdget(args->fd);
1425 0 : if (IS_ERR(ev_fd_ctx))
1426 0 : return PTR_ERR(ev_fd_ctx);
1427 :
1428 0 : entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1429 0 : if (!entry) {
1430 0 : eventfd_ctx_put(ev_fd_ctx);
1431 0 : return -ENOMEM;
1432 : }
1433 0 : entry->syncobj = syncobj;
1434 0 : entry->ev_fd_ctx = ev_fd_ctx;
1435 0 : entry->point = args->point;
1436 0 : entry->flags = args->flags;
1437 :
1438 0 : drm_syncobj_add_eventfd(syncobj, entry);
1439 0 : drm_syncobj_put(syncobj);
1440 :
1441 0 : return 0;
1442 : }
1443 :
1444 : int
1445 0 : drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1446 : struct drm_file *file_private)
1447 : {
1448 0 : struct drm_syncobj_array *args = data;
1449 : struct drm_syncobj **syncobjs;
1450 : uint32_t i;
1451 : int ret;
1452 :
1453 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1454 : return -EOPNOTSUPP;
1455 :
1456 0 : if (args->pad != 0)
1457 : return -EINVAL;
1458 :
1459 0 : if (args->count_handles == 0)
1460 : return -EINVAL;
1461 :
1462 0 : ret = drm_syncobj_array_find(file_private,
1463 0 : u64_to_user_ptr(args->handles),
1464 : args->count_handles,
1465 : &syncobjs);
1466 0 : if (ret < 0)
1467 : return ret;
1468 :
1469 0 : for (i = 0; i < args->count_handles; i++)
1470 0 : drm_syncobj_replace_fence(syncobjs[i], NULL);
1471 :
1472 0 : drm_syncobj_array_free(syncobjs, args->count_handles);
1473 :
1474 0 : return 0;
1475 : }
1476 :
1477 : int
1478 0 : drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1479 : struct drm_file *file_private)
1480 : {
1481 0 : struct drm_syncobj_array *args = data;
1482 : struct drm_syncobj **syncobjs;
1483 : uint32_t i;
1484 : int ret;
1485 :
1486 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1487 : return -EOPNOTSUPP;
1488 :
1489 0 : if (args->pad != 0)
1490 : return -EINVAL;
1491 :
1492 0 : if (args->count_handles == 0)
1493 : return -EINVAL;
1494 :
1495 0 : ret = drm_syncobj_array_find(file_private,
1496 0 : u64_to_user_ptr(args->handles),
1497 : args->count_handles,
1498 : &syncobjs);
1499 0 : if (ret < 0)
1500 : return ret;
1501 :
1502 0 : for (i = 0; i < args->count_handles; i++) {
1503 0 : ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1504 0 : if (ret < 0)
1505 : break;
1506 : }
1507 :
1508 0 : drm_syncobj_array_free(syncobjs, args->count_handles);
1509 :
1510 0 : return ret;
1511 : }
1512 :
1513 : int
1514 0 : drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1515 : struct drm_file *file_private)
1516 : {
1517 0 : struct drm_syncobj_timeline_array *args = data;
1518 : struct drm_syncobj **syncobjs;
1519 : struct dma_fence_chain **chains;
1520 : uint64_t *points;
1521 : uint32_t i, j;
1522 : int ret;
1523 :
1524 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1525 : return -EOPNOTSUPP;
1526 :
1527 0 : if (args->flags != 0)
1528 : return -EINVAL;
1529 :
1530 0 : if (args->count_handles == 0)
1531 : return -EINVAL;
1532 :
1533 0 : ret = drm_syncobj_array_find(file_private,
1534 0 : u64_to_user_ptr(args->handles),
1535 : args->count_handles,
1536 : &syncobjs);
1537 0 : if (ret < 0)
1538 : return ret;
1539 :
1540 0 : points = kmalloc_array(args->count_handles, sizeof(*points),
1541 : GFP_KERNEL);
1542 0 : if (!points) {
1543 : ret = -ENOMEM;
1544 : goto out;
1545 : }
1546 0 : if (!u64_to_user_ptr(args->points)) {
1547 0 : memset(points, 0, args->count_handles * sizeof(uint64_t));
1548 0 : } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1549 0 : sizeof(uint64_t) * args->count_handles)) {
1550 : ret = -EFAULT;
1551 : goto err_points;
1552 : }
1553 :
1554 0 : chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1555 0 : if (!chains) {
1556 : ret = -ENOMEM;
1557 : goto err_points;
1558 : }
1559 0 : for (i = 0; i < args->count_handles; i++) {
1560 0 : chains[i] = dma_fence_chain_alloc();
1561 0 : if (!chains[i]) {
1562 0 : for (j = 0; j < i; j++)
1563 0 : dma_fence_chain_free(chains[j]);
1564 : ret = -ENOMEM;
1565 : goto err_chains;
1566 : }
1567 : }
1568 :
1569 0 : for (i = 0; i < args->count_handles; i++) {
1570 0 : struct dma_fence *fence = dma_fence_get_stub();
1571 :
1572 0 : drm_syncobj_add_point(syncobjs[i], chains[i],
1573 0 : fence, points[i]);
1574 0 : dma_fence_put(fence);
1575 : }
1576 : err_chains:
1577 0 : kfree(chains);
1578 : err_points:
1579 0 : kfree(points);
1580 : out:
1581 0 : drm_syncobj_array_free(syncobjs, args->count_handles);
1582 :
1583 0 : return ret;
1584 : }
1585 :
1586 0 : int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1587 : struct drm_file *file_private)
1588 : {
1589 0 : struct drm_syncobj_timeline_array *args = data;
1590 : struct drm_syncobj **syncobjs;
1591 0 : uint64_t __user *points = u64_to_user_ptr(args->points);
1592 : uint32_t i;
1593 : int ret;
1594 :
1595 0 : if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1596 : return -EOPNOTSUPP;
1597 :
1598 0 : if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1599 : return -EINVAL;
1600 :
1601 0 : if (args->count_handles == 0)
1602 : return -EINVAL;
1603 :
1604 0 : ret = drm_syncobj_array_find(file_private,
1605 0 : u64_to_user_ptr(args->handles),
1606 : args->count_handles,
1607 : &syncobjs);
1608 0 : if (ret < 0)
1609 : return ret;
1610 :
1611 0 : for (i = 0; i < args->count_handles; i++) {
1612 : struct dma_fence_chain *chain;
1613 : struct dma_fence *fence;
1614 : uint64_t point;
1615 :
1616 0 : fence = drm_syncobj_fence_get(syncobjs[i]);
1617 0 : chain = to_dma_fence_chain(fence);
1618 0 : if (chain) {
1619 0 : struct dma_fence *iter, *last_signaled =
1620 : dma_fence_get(fence);
1621 :
1622 0 : if (args->flags &
1623 : DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1624 0 : point = fence->seqno;
1625 : } else {
1626 0 : dma_fence_chain_for_each(iter, fence) {
1627 0 : if (iter->context != fence->context) {
1628 : dma_fence_put(iter);
1629 : /* It is most likely that timeline has
1630 : * unorder points. */
1631 : break;
1632 : }
1633 0 : dma_fence_put(last_signaled);
1634 0 : last_signaled = dma_fence_get(iter);
1635 : }
1636 0 : point = dma_fence_is_signaled(last_signaled) ?
1637 0 : last_signaled->seqno :
1638 0 : to_dma_fence_chain(last_signaled)->prev_seqno;
1639 : }
1640 : dma_fence_put(last_signaled);
1641 : } else {
1642 0 : point = 0;
1643 : }
1644 0 : dma_fence_put(fence);
1645 0 : ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1646 0 : ret = ret ? -EFAULT : 0;
1647 0 : if (ret)
1648 : break;
1649 : }
1650 0 : drm_syncobj_array_free(syncobjs, args->count_handles);
1651 :
1652 0 : return ret;
1653 : }
|
