Line data Source code
1 : /*
2 : * Copyright © 2008 Intel Corporation
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a
5 : * copy of this software and associated documentation files (the "Software"),
6 : * to deal in the Software without restriction, including without limitation
7 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : * and/or sell copies of the Software, and to permit persons to whom the
9 : * Software is furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice (including the next
12 : * paragraph) shall be included in all copies or substantial portions of the
13 : * Software.
14 : *
15 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 : * IN THE SOFTWARE.
22 : *
23 : * Authors:
24 : * Eric Anholt <eric@anholt.net>
25 : *
26 : */
27 :
28 : #include <linux/dma-buf.h>
29 : #include <linux/file.h>
30 : #include <linux/fs.h>
31 : #include <linux/iosys-map.h>
32 : #include <linux/mem_encrypt.h>
33 : #include <linux/mm.h>
34 : #include <linux/mman.h>
35 : #include <linux/module.h>
36 : #include <linux/pagemap.h>
37 : #include <linux/pagevec.h>
38 : #include <linux/shmem_fs.h>
39 : #include <linux/slab.h>
40 : #include <linux/string_helpers.h>
41 : #include <linux/types.h>
42 : #include <linux/uaccess.h>
43 :
44 : #include <drm/drm.h>
45 : #include <drm/drm_device.h>
46 : #include <drm/drm_drv.h>
47 : #include <drm/drm_file.h>
48 : #include <drm/drm_gem.h>
49 : #include <drm/drm_managed.h>
50 : #include <drm/drm_print.h>
51 : #include <drm/drm_vma_manager.h>
52 :
53 : #include "drm_internal.h"
54 :
55 : /** @file drm_gem.c
56 : *
57 : * This file provides some of the base ioctls and library routines for
58 : * the graphics memory manager implemented by each device driver.
59 : *
60 : * Because various devices have different requirements in terms of
61 : * synchronization and migration strategies, implementing that is left up to
62 : * the driver, and all that the general API provides should be generic --
63 : * allocating objects, reading/writing data with the cpu, freeing objects.
64 : * Even there, platform-dependent optimizations for reading/writing data with
65 : * the CPU mean we'll likely hook those out to driver-specific calls. However,
66 : * the DRI2 implementation wants to have at least allocate/mmap be generic.
67 : *
68 : * The goal was to have swap-backed object allocation managed through
69 : * struct file. However, file descriptors as handles to a struct file have
70 : * two major failings:
71 : * - Process limits prevent more than 1024 or so being used at a time by
72 : * default.
73 : * - Inability to allocate high fds will aggravate the X Server's select()
74 : * handling, and likely that of many GL client applications as well.
75 : *
76 : * This led to a plan of using our own integer IDs (called handles, following
77 : * DRM terminology) to mimic fds, and implement the fd syscalls we need as
78 : * ioctls. The objects themselves will still include the struct file so
79 : * that we can transition to fds if the required kernel infrastructure shows
80 : * up at a later date, and as our interface with shmfs for memory allocation.
81 : */
82 :
83 : static void
84 0 : drm_gem_init_release(struct drm_device *dev, void *ptr)
85 : {
86 0 : drm_vma_offset_manager_destroy(dev->vma_offset_manager);
87 0 : }
88 :
89 : /**
90 : * drm_gem_init - Initialize the GEM device fields
91 : * @dev: drm_devic structure to initialize
92 : */
93 : int
94 0 : drm_gem_init(struct drm_device *dev)
95 : {
96 : struct drm_vma_offset_manager *vma_offset_manager;
97 :
98 0 : mutex_init(&dev->object_name_lock);
99 0 : idr_init_base(&dev->object_name_idr, 1);
100 :
101 0 : vma_offset_manager = drmm_kzalloc(dev, sizeof(*vma_offset_manager),
102 : GFP_KERNEL);
103 0 : if (!vma_offset_manager) {
104 0 : DRM_ERROR("out of memory\n");
105 0 : return -ENOMEM;
106 : }
107 :
108 0 : dev->vma_offset_manager = vma_offset_manager;
109 0 : drm_vma_offset_manager_init(vma_offset_manager,
110 : DRM_FILE_PAGE_OFFSET_START,
111 : DRM_FILE_PAGE_OFFSET_SIZE);
112 :
113 0 : return drmm_add_action(dev, drm_gem_init_release, NULL);
114 : }
115 :
116 : /**
117 : * drm_gem_object_init - initialize an allocated shmem-backed GEM object
118 : * @dev: drm_device the object should be initialized for
119 : * @obj: drm_gem_object to initialize
120 : * @size: object size
121 : *
122 : * Initialize an already allocated GEM object of the specified size with
123 : * shmfs backing store.
124 : */
125 0 : int drm_gem_object_init(struct drm_device *dev,
126 : struct drm_gem_object *obj, size_t size)
127 : {
128 : struct file *filp;
129 :
130 0 : drm_gem_private_object_init(dev, obj, size);
131 :
132 0 : filp = shmem_file_setup("drm mm object", size, VM_NORESERVE);
133 0 : if (IS_ERR(filp))
134 0 : return PTR_ERR(filp);
135 :
136 0 : obj->filp = filp;
137 :
138 0 : return 0;
139 : }
140 : EXPORT_SYMBOL(drm_gem_object_init);
141 :
142 : /**
143 : * drm_gem_private_object_init - initialize an allocated private GEM object
144 : * @dev: drm_device the object should be initialized for
145 : * @obj: drm_gem_object to initialize
146 : * @size: object size
147 : *
148 : * Initialize an already allocated GEM object of the specified size with
149 : * no GEM provided backing store. Instead the caller is responsible for
150 : * backing the object and handling it.
151 : */
152 0 : void drm_gem_private_object_init(struct drm_device *dev,
153 : struct drm_gem_object *obj, size_t size)
154 : {
155 0 : BUG_ON((size & (PAGE_SIZE - 1)) != 0);
156 :
157 0 : obj->dev = dev;
158 0 : obj->filp = NULL;
159 :
160 0 : kref_init(&obj->refcount);
161 0 : obj->handle_count = 0;
162 0 : obj->size = size;
163 0 : dma_resv_init(&obj->_resv);
164 0 : if (!obj->resv)
165 0 : obj->resv = &obj->_resv;
166 :
167 0 : if (drm_core_check_feature(dev, DRIVER_GEM_GPUVA))
168 : drm_gem_gpuva_init(obj);
169 :
170 0 : drm_vma_node_reset(&obj->vma_node);
171 0 : INIT_LIST_HEAD(&obj->lru_node);
172 0 : }
173 : EXPORT_SYMBOL(drm_gem_private_object_init);
174 :
175 : /**
176 : * drm_gem_private_object_fini - Finalize a failed drm_gem_object
177 : * @obj: drm_gem_object
178 : *
179 : * Uninitialize an already allocated GEM object when it initialized failed
180 : */
181 0 : void drm_gem_private_object_fini(struct drm_gem_object *obj)
182 : {
183 0 : WARN_ON(obj->dma_buf);
184 :
185 0 : dma_resv_fini(&obj->_resv);
186 0 : }
187 : EXPORT_SYMBOL(drm_gem_private_object_fini);
188 :
189 : /**
190 : * drm_gem_object_handle_free - release resources bound to userspace handles
191 : * @obj: GEM object to clean up.
192 : *
193 : * Called after the last handle to the object has been closed
194 : *
195 : * Removes any name for the object. Note that this must be
196 : * called before drm_gem_object_free or we'll be touching
197 : * freed memory
198 : */
199 : static void drm_gem_object_handle_free(struct drm_gem_object *obj)
200 : {
201 0 : struct drm_device *dev = obj->dev;
202 :
203 : /* Remove any name for this object */
204 0 : if (obj->name) {
205 0 : idr_remove(&dev->object_name_idr, obj->name);
206 0 : obj->name = 0;
207 : }
208 : }
209 :
210 : static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)
211 : {
212 : /* Unbreak the reference cycle if we have an exported dma_buf. */
213 0 : if (obj->dma_buf) {
214 0 : dma_buf_put(obj->dma_buf);
215 0 : obj->dma_buf = NULL;
216 : }
217 : }
218 :
219 : static void
220 0 : drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)
221 : {
222 0 : struct drm_device *dev = obj->dev;
223 0 : bool final = false;
224 :
225 0 : if (WARN_ON(READ_ONCE(obj->handle_count) == 0))
226 : return;
227 :
228 : /*
229 : * Must bump handle count first as this may be the last
230 : * ref, in which case the object would disappear before we
231 : * checked for a name
232 : */
233 :
234 0 : mutex_lock(&dev->object_name_lock);
235 0 : if (--obj->handle_count == 0) {
236 0 : drm_gem_object_handle_free(obj);
237 0 : drm_gem_object_exported_dma_buf_free(obj);
238 : final = true;
239 : }
240 0 : mutex_unlock(&dev->object_name_lock);
241 :
242 0 : if (final)
243 : drm_gem_object_put(obj);
244 : }
245 :
246 : /*
247 : * Called at device or object close to release the file's
248 : * handle references on objects.
249 : */
250 : static int
251 0 : drm_gem_object_release_handle(int id, void *ptr, void *data)
252 : {
253 0 : struct drm_file *file_priv = data;
254 0 : struct drm_gem_object *obj = ptr;
255 :
256 0 : if (obj->funcs->close)
257 0 : obj->funcs->close(obj, file_priv);
258 :
259 0 : drm_prime_remove_buf_handle(&file_priv->prime, id);
260 0 : drm_vma_node_revoke(&obj->vma_node, file_priv);
261 :
262 0 : drm_gem_object_handle_put_unlocked(obj);
263 :
264 0 : return 0;
265 : }
266 :
267 : /**
268 : * drm_gem_handle_delete - deletes the given file-private handle
269 : * @filp: drm file-private structure to use for the handle look up
270 : * @handle: userspace handle to delete
271 : *
272 : * Removes the GEM handle from the @filp lookup table which has been added with
273 : * drm_gem_handle_create(). If this is the last handle also cleans up linked
274 : * resources like GEM names.
275 : */
276 : int
277 0 : drm_gem_handle_delete(struct drm_file *filp, u32 handle)
278 : {
279 : struct drm_gem_object *obj;
280 :
281 0 : spin_lock(&filp->table_lock);
282 :
283 : /* Check if we currently have a reference on the object */
284 0 : obj = idr_replace(&filp->object_idr, NULL, handle);
285 0 : spin_unlock(&filp->table_lock);
286 0 : if (IS_ERR_OR_NULL(obj))
287 : return -EINVAL;
288 :
289 : /* Release driver's reference and decrement refcount. */
290 0 : drm_gem_object_release_handle(handle, obj, filp);
291 :
292 : /* And finally make the handle available for future allocations. */
293 0 : spin_lock(&filp->table_lock);
294 0 : idr_remove(&filp->object_idr, handle);
295 0 : spin_unlock(&filp->table_lock);
296 :
297 0 : return 0;
298 : }
299 : EXPORT_SYMBOL(drm_gem_handle_delete);
300 :
301 : /**
302 : * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object
303 : * @file: drm file-private structure containing the gem object
304 : * @dev: corresponding drm_device
305 : * @handle: gem object handle
306 : * @offset: return location for the fake mmap offset
307 : *
308 : * This implements the &drm_driver.dumb_map_offset kms driver callback for
309 : * drivers which use gem to manage their backing storage.
310 : *
311 : * Returns:
312 : * 0 on success or a negative error code on failure.
313 : */
314 0 : int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
315 : u32 handle, u64 *offset)
316 : {
317 : struct drm_gem_object *obj;
318 : int ret;
319 :
320 0 : obj = drm_gem_object_lookup(file, handle);
321 0 : if (!obj)
322 : return -ENOENT;
323 :
324 : /* Don't allow imported objects to be mapped */
325 0 : if (obj->import_attach) {
326 : ret = -EINVAL;
327 : goto out;
328 : }
329 :
330 0 : ret = drm_gem_create_mmap_offset(obj);
331 0 : if (ret)
332 : goto out;
333 :
334 0 : *offset = drm_vma_node_offset_addr(&obj->vma_node);
335 : out:
336 0 : drm_gem_object_put(obj);
337 :
338 0 : return ret;
339 : }
340 : EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
341 :
342 : /**
343 : * drm_gem_handle_create_tail - internal functions to create a handle
344 : * @file_priv: drm file-private structure to register the handle for
345 : * @obj: object to register
346 : * @handlep: pointer to return the created handle to the caller
347 : *
348 : * This expects the &drm_device.object_name_lock to be held already and will
349 : * drop it before returning. Used to avoid races in establishing new handles
350 : * when importing an object from either an flink name or a dma-buf.
351 : *
352 : * Handles must be release again through drm_gem_handle_delete(). This is done
353 : * when userspace closes @file_priv for all attached handles, or through the
354 : * GEM_CLOSE ioctl for individual handles.
355 : */
356 : int
357 0 : drm_gem_handle_create_tail(struct drm_file *file_priv,
358 : struct drm_gem_object *obj,
359 : u32 *handlep)
360 : {
361 0 : struct drm_device *dev = obj->dev;
362 : u32 handle;
363 : int ret;
364 :
365 0 : WARN_ON(!mutex_is_locked(&dev->object_name_lock));
366 0 : if (obj->handle_count++ == 0)
367 : drm_gem_object_get(obj);
368 :
369 : /*
370 : * Get the user-visible handle using idr. Preload and perform
371 : * allocation under our spinlock.
372 : */
373 0 : idr_preload(GFP_KERNEL);
374 0 : spin_lock(&file_priv->table_lock);
375 :
376 0 : ret = idr_alloc(&file_priv->object_idr, obj, 1, 0, GFP_NOWAIT);
377 :
378 0 : spin_unlock(&file_priv->table_lock);
379 : idr_preload_end();
380 :
381 0 : mutex_unlock(&dev->object_name_lock);
382 0 : if (ret < 0)
383 : goto err_unref;
384 :
385 0 : handle = ret;
386 :
387 0 : ret = drm_vma_node_allow(&obj->vma_node, file_priv);
388 0 : if (ret)
389 : goto err_remove;
390 :
391 0 : if (obj->funcs->open) {
392 0 : ret = obj->funcs->open(obj, file_priv);
393 0 : if (ret)
394 : goto err_revoke;
395 : }
396 :
397 0 : *handlep = handle;
398 0 : return 0;
399 :
400 : err_revoke:
401 0 : drm_vma_node_revoke(&obj->vma_node, file_priv);
402 : err_remove:
403 0 : spin_lock(&file_priv->table_lock);
404 0 : idr_remove(&file_priv->object_idr, handle);
405 0 : spin_unlock(&file_priv->table_lock);
406 : err_unref:
407 0 : drm_gem_object_handle_put_unlocked(obj);
408 0 : return ret;
409 : }
410 :
411 : /**
412 : * drm_gem_handle_create - create a gem handle for an object
413 : * @file_priv: drm file-private structure to register the handle for
414 : * @obj: object to register
415 : * @handlep: pointer to return the created handle to the caller
416 : *
417 : * Create a handle for this object. This adds a handle reference to the object,
418 : * which includes a regular reference count. Callers will likely want to
419 : * dereference the object afterwards.
420 : *
421 : * Since this publishes @obj to userspace it must be fully set up by this point,
422 : * drivers must call this last in their buffer object creation callbacks.
423 : */
424 0 : int drm_gem_handle_create(struct drm_file *file_priv,
425 : struct drm_gem_object *obj,
426 : u32 *handlep)
427 : {
428 0 : mutex_lock(&obj->dev->object_name_lock);
429 :
430 0 : return drm_gem_handle_create_tail(file_priv, obj, handlep);
431 : }
432 : EXPORT_SYMBOL(drm_gem_handle_create);
433 :
434 :
435 : /**
436 : * drm_gem_free_mmap_offset - release a fake mmap offset for an object
437 : * @obj: obj in question
438 : *
439 : * This routine frees fake offsets allocated by drm_gem_create_mmap_offset().
440 : *
441 : * Note that drm_gem_object_release() already calls this function, so drivers
442 : * don't have to take care of releasing the mmap offset themselves when freeing
443 : * the GEM object.
444 : */
445 : void
446 0 : drm_gem_free_mmap_offset(struct drm_gem_object *obj)
447 : {
448 0 : struct drm_device *dev = obj->dev;
449 :
450 0 : drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node);
451 0 : }
452 : EXPORT_SYMBOL(drm_gem_free_mmap_offset);
453 :
454 : /**
455 : * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object
456 : * @obj: obj in question
457 : * @size: the virtual size
458 : *
459 : * GEM memory mapping works by handing back to userspace a fake mmap offset
460 : * it can use in a subsequent mmap(2) call. The DRM core code then looks
461 : * up the object based on the offset and sets up the various memory mapping
462 : * structures.
463 : *
464 : * This routine allocates and attaches a fake offset for @obj, in cases where
465 : * the virtual size differs from the physical size (ie. &drm_gem_object.size).
466 : * Otherwise just use drm_gem_create_mmap_offset().
467 : *
468 : * This function is idempotent and handles an already allocated mmap offset
469 : * transparently. Drivers do not need to check for this case.
470 : */
471 : int
472 0 : drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size)
473 : {
474 0 : struct drm_device *dev = obj->dev;
475 :
476 0 : return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node,
477 : size / PAGE_SIZE);
478 : }
479 : EXPORT_SYMBOL(drm_gem_create_mmap_offset_size);
480 :
481 : /**
482 : * drm_gem_create_mmap_offset - create a fake mmap offset for an object
483 : * @obj: obj in question
484 : *
485 : * GEM memory mapping works by handing back to userspace a fake mmap offset
486 : * it can use in a subsequent mmap(2) call. The DRM core code then looks
487 : * up the object based on the offset and sets up the various memory mapping
488 : * structures.
489 : *
490 : * This routine allocates and attaches a fake offset for @obj.
491 : *
492 : * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release
493 : * the fake offset again.
494 : */
495 0 : int drm_gem_create_mmap_offset(struct drm_gem_object *obj)
496 : {
497 0 : return drm_gem_create_mmap_offset_size(obj, obj->size);
498 : }
499 : EXPORT_SYMBOL(drm_gem_create_mmap_offset);
500 :
501 : /*
502 : * Move folios to appropriate lru and release the folios, decrementing the
503 : * ref count of those folios.
504 : */
505 : static void drm_gem_check_release_batch(struct folio_batch *fbatch)
506 : {
507 0 : check_move_unevictable_folios(fbatch);
508 0 : __folio_batch_release(fbatch);
509 0 : cond_resched();
510 : }
511 :
512 : /**
513 : * drm_gem_get_pages - helper to allocate backing pages for a GEM object
514 : * from shmem
515 : * @obj: obj in question
516 : *
517 : * This reads the page-array of the shmem-backing storage of the given gem
518 : * object. An array of pages is returned. If a page is not allocated or
519 : * swapped-out, this will allocate/swap-in the required pages. Note that the
520 : * whole object is covered by the page-array and pinned in memory.
521 : *
522 : * Use drm_gem_put_pages() to release the array and unpin all pages.
523 : *
524 : * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()).
525 : * If you require other GFP-masks, you have to do those allocations yourself.
526 : *
527 : * Note that you are not allowed to change gfp-zones during runtime. That is,
528 : * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as
529 : * set during initialization. If you have special zone constraints, set them
530 : * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care
531 : * to keep pages in the required zone during swap-in.
532 : *
533 : * This function is only valid on objects initialized with
534 : * drm_gem_object_init(), but not for those initialized with
535 : * drm_gem_private_object_init() only.
536 : */
537 0 : struct page **drm_gem_get_pages(struct drm_gem_object *obj)
538 : {
539 : struct address_space *mapping;
540 : struct page **pages;
541 : struct folio *folio;
542 : struct folio_batch fbatch;
543 : int i, j, npages;
544 :
545 0 : if (WARN_ON(!obj->filp))
546 : return ERR_PTR(-EINVAL);
547 :
548 : /* This is the shared memory object that backs the GEM resource */
549 0 : mapping = obj->filp->f_mapping;
550 :
551 : /* We already BUG_ON() for non-page-aligned sizes in
552 : * drm_gem_object_init(), so we should never hit this unless
553 : * driver author is doing something really wrong:
554 : */
555 0 : WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
556 :
557 0 : npages = obj->size >> PAGE_SHIFT;
558 :
559 0 : pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
560 0 : if (pages == NULL)
561 : return ERR_PTR(-ENOMEM);
562 :
563 0 : mapping_set_unevictable(mapping);
564 :
565 0 : i = 0;
566 0 : while (i < npages) {
567 0 : folio = shmem_read_folio_gfp(mapping, i,
568 : mapping_gfp_mask(mapping));
569 0 : if (IS_ERR(folio))
570 : goto fail;
571 0 : for (j = 0; j < folio_nr_pages(folio); j++, i++)
572 0 : pages[i] = folio_file_page(folio, i);
573 :
574 : /* Make sure shmem keeps __GFP_DMA32 allocated pages in the
575 : * correct region during swapin. Note that this requires
576 : * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping)
577 : * so shmem can relocate pages during swapin if required.
578 : */
579 0 : BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) &&
580 : (folio_pfn(folio) >= 0x00100000UL));
581 : }
582 :
583 : return pages;
584 :
585 : fail:
586 0 : mapping_clear_unevictable(mapping);
587 0 : folio_batch_init(&fbatch);
588 0 : j = 0;
589 0 : while (j < i) {
590 0 : struct folio *f = page_folio(pages[j]);
591 0 : if (!folio_batch_add(&fbatch, f))
592 : drm_gem_check_release_batch(&fbatch);
593 0 : j += folio_nr_pages(f);
594 : }
595 0 : if (fbatch.nr)
596 : drm_gem_check_release_batch(&fbatch);
597 :
598 0 : kvfree(pages);
599 0 : return ERR_CAST(folio);
600 : }
601 : EXPORT_SYMBOL(drm_gem_get_pages);
602 :
603 : /**
604 : * drm_gem_put_pages - helper to free backing pages for a GEM object
605 : * @obj: obj in question
606 : * @pages: pages to free
607 : * @dirty: if true, pages will be marked as dirty
608 : * @accessed: if true, the pages will be marked as accessed
609 : */
610 0 : void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
611 : bool dirty, bool accessed)
612 : {
613 : int i, npages;
614 : struct address_space *mapping;
615 : struct folio_batch fbatch;
616 :
617 0 : mapping = file_inode(obj->filp)->i_mapping;
618 0 : mapping_clear_unevictable(mapping);
619 :
620 : /* We already BUG_ON() for non-page-aligned sizes in
621 : * drm_gem_object_init(), so we should never hit this unless
622 : * driver author is doing something really wrong:
623 : */
624 0 : WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
625 :
626 0 : npages = obj->size >> PAGE_SHIFT;
627 :
628 0 : folio_batch_init(&fbatch);
629 0 : for (i = 0; i < npages; i++) {
630 : struct folio *folio;
631 :
632 0 : if (!pages[i])
633 0 : continue;
634 0 : folio = page_folio(pages[i]);
635 :
636 0 : if (dirty)
637 0 : folio_mark_dirty(folio);
638 :
639 0 : if (accessed)
640 0 : folio_mark_accessed(folio);
641 :
642 : /* Undo the reference we took when populating the table */
643 0 : if (!folio_batch_add(&fbatch, folio))
644 : drm_gem_check_release_batch(&fbatch);
645 0 : i += folio_nr_pages(folio) - 1;
646 : }
647 0 : if (folio_batch_count(&fbatch))
648 : drm_gem_check_release_batch(&fbatch);
649 :
650 0 : kvfree(pages);
651 0 : }
652 : EXPORT_SYMBOL(drm_gem_put_pages);
653 :
654 0 : static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
655 : struct drm_gem_object **objs)
656 : {
657 0 : int i, ret = 0;
658 : struct drm_gem_object *obj;
659 :
660 0 : spin_lock(&filp->table_lock);
661 :
662 0 : for (i = 0; i < count; i++) {
663 : /* Check if we currently have a reference on the object */
664 0 : obj = idr_find(&filp->object_idr, handle[i]);
665 0 : if (!obj) {
666 : ret = -ENOENT;
667 : break;
668 : }
669 0 : drm_gem_object_get(obj);
670 0 : objs[i] = obj;
671 : }
672 0 : spin_unlock(&filp->table_lock);
673 :
674 0 : return ret;
675 : }
676 :
677 : /**
678 : * drm_gem_objects_lookup - look up GEM objects from an array of handles
679 : * @filp: DRM file private date
680 : * @bo_handles: user pointer to array of userspace handle
681 : * @count: size of handle array
682 : * @objs_out: returned pointer to array of drm_gem_object pointers
683 : *
684 : * Takes an array of userspace handles and returns a newly allocated array of
685 : * GEM objects.
686 : *
687 : * For a single handle lookup, use drm_gem_object_lookup().
688 : *
689 : * Returns:
690 : *
691 : * @objs filled in with GEM object pointers. Returned GEM objects need to be
692 : * released with drm_gem_object_put(). -ENOENT is returned on a lookup
693 : * failure. 0 is returned on success.
694 : *
695 : */
696 0 : int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
697 : int count, struct drm_gem_object ***objs_out)
698 : {
699 : int ret;
700 : u32 *handles;
701 : struct drm_gem_object **objs;
702 :
703 0 : if (!count)
704 : return 0;
705 :
706 0 : objs = kvmalloc_array(count, sizeof(struct drm_gem_object *),
707 : GFP_KERNEL | __GFP_ZERO);
708 0 : if (!objs)
709 : return -ENOMEM;
710 :
711 0 : *objs_out = objs;
712 :
713 0 : handles = kvmalloc_array(count, sizeof(u32), GFP_KERNEL);
714 0 : if (!handles) {
715 : ret = -ENOMEM;
716 : goto out;
717 : }
718 :
719 0 : if (copy_from_user(handles, bo_handles, count * sizeof(u32))) {
720 0 : ret = -EFAULT;
721 0 : DRM_DEBUG("Failed to copy in GEM handles\n");
722 0 : goto out;
723 : }
724 :
725 0 : ret = objects_lookup(filp, handles, count, objs);
726 : out:
727 0 : kvfree(handles);
728 0 : return ret;
729 :
730 : }
731 : EXPORT_SYMBOL(drm_gem_objects_lookup);
732 :
733 : /**
734 : * drm_gem_object_lookup - look up a GEM object from its handle
735 : * @filp: DRM file private date
736 : * @handle: userspace handle
737 : *
738 : * Returns:
739 : *
740 : * A reference to the object named by the handle if such exists on @filp, NULL
741 : * otherwise.
742 : *
743 : * If looking up an array of handles, use drm_gem_objects_lookup().
744 : */
745 : struct drm_gem_object *
746 0 : drm_gem_object_lookup(struct drm_file *filp, u32 handle)
747 : {
748 0 : struct drm_gem_object *obj = NULL;
749 :
750 0 : objects_lookup(filp, &handle, 1, &obj);
751 0 : return obj;
752 : }
753 : EXPORT_SYMBOL(drm_gem_object_lookup);
754 :
755 : /**
756 : * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects
757 : * shared and/or exclusive fences.
758 : * @filep: DRM file private date
759 : * @handle: userspace handle
760 : * @wait_all: if true, wait on all fences, else wait on just exclusive fence
761 : * @timeout: timeout value in jiffies or zero to return immediately
762 : *
763 : * Returns:
764 : *
765 : * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
766 : * greater than 0 on success.
767 : */
768 0 : long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle,
769 : bool wait_all, unsigned long timeout)
770 : {
771 : long ret;
772 : struct drm_gem_object *obj;
773 :
774 0 : obj = drm_gem_object_lookup(filep, handle);
775 0 : if (!obj) {
776 0 : DRM_DEBUG("Failed to look up GEM BO %d\n", handle);
777 0 : return -EINVAL;
778 : }
779 :
780 0 : ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all),
781 : true, timeout);
782 0 : if (ret == 0)
783 : ret = -ETIME;
784 0 : else if (ret > 0)
785 0 : ret = 0;
786 :
787 0 : drm_gem_object_put(obj);
788 :
789 0 : return ret;
790 : }
791 : EXPORT_SYMBOL(drm_gem_dma_resv_wait);
792 :
793 : /**
794 : * drm_gem_close_ioctl - implementation of the GEM_CLOSE ioctl
795 : * @dev: drm_device
796 : * @data: ioctl data
797 : * @file_priv: drm file-private structure
798 : *
799 : * Releases the handle to an mm object.
800 : */
801 : int
802 0 : drm_gem_close_ioctl(struct drm_device *dev, void *data,
803 : struct drm_file *file_priv)
804 : {
805 0 : struct drm_gem_close *args = data;
806 : int ret;
807 :
808 0 : if (!drm_core_check_feature(dev, DRIVER_GEM))
809 : return -EOPNOTSUPP;
810 :
811 0 : ret = drm_gem_handle_delete(file_priv, args->handle);
812 :
813 0 : return ret;
814 : }
815 :
816 : /**
817 : * drm_gem_flink_ioctl - implementation of the GEM_FLINK ioctl
818 : * @dev: drm_device
819 : * @data: ioctl data
820 : * @file_priv: drm file-private structure
821 : *
822 : * Create a global name for an object, returning the name.
823 : *
824 : * Note that the name does not hold a reference; when the object
825 : * is freed, the name goes away.
826 : */
827 : int
828 0 : drm_gem_flink_ioctl(struct drm_device *dev, void *data,
829 : struct drm_file *file_priv)
830 : {
831 0 : struct drm_gem_flink *args = data;
832 : struct drm_gem_object *obj;
833 : int ret;
834 :
835 0 : if (!drm_core_check_feature(dev, DRIVER_GEM))
836 : return -EOPNOTSUPP;
837 :
838 0 : obj = drm_gem_object_lookup(file_priv, args->handle);
839 0 : if (obj == NULL)
840 : return -ENOENT;
841 :
842 0 : mutex_lock(&dev->object_name_lock);
843 : /* prevent races with concurrent gem_close. */
844 0 : if (obj->handle_count == 0) {
845 : ret = -ENOENT;
846 : goto err;
847 : }
848 :
849 0 : if (!obj->name) {
850 0 : ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);
851 0 : if (ret < 0)
852 : goto err;
853 :
854 0 : obj->name = ret;
855 : }
856 :
857 0 : args->name = (uint64_t) obj->name;
858 0 : ret = 0;
859 :
860 : err:
861 0 : mutex_unlock(&dev->object_name_lock);
862 0 : drm_gem_object_put(obj);
863 0 : return ret;
864 : }
865 :
866 : /**
867 : * drm_gem_open_ioctl - implementation of the GEM_OPEN ioctl
868 : * @dev: drm_device
869 : * @data: ioctl data
870 : * @file_priv: drm file-private structure
871 : *
872 : * Open an object using the global name, returning a handle and the size.
873 : *
874 : * This handle (of course) holds a reference to the object, so the object
875 : * will not go away until the handle is deleted.
876 : */
877 : int
878 0 : drm_gem_open_ioctl(struct drm_device *dev, void *data,
879 : struct drm_file *file_priv)
880 : {
881 0 : struct drm_gem_open *args = data;
882 : struct drm_gem_object *obj;
883 : int ret;
884 : u32 handle;
885 :
886 0 : if (!drm_core_check_feature(dev, DRIVER_GEM))
887 : return -EOPNOTSUPP;
888 :
889 0 : mutex_lock(&dev->object_name_lock);
890 0 : obj = idr_find(&dev->object_name_idr, (int) args->name);
891 0 : if (obj) {
892 0 : drm_gem_object_get(obj);
893 : } else {
894 0 : mutex_unlock(&dev->object_name_lock);
895 0 : return -ENOENT;
896 : }
897 :
898 : /* drm_gem_handle_create_tail unlocks dev->object_name_lock. */
899 0 : ret = drm_gem_handle_create_tail(file_priv, obj, &handle);
900 0 : if (ret)
901 : goto err;
902 :
903 0 : args->handle = handle;
904 0 : args->size = obj->size;
905 :
906 : err:
907 0 : drm_gem_object_put(obj);
908 0 : return ret;
909 : }
910 :
911 : /**
912 : * drm_gem_open - initializes GEM file-private structures at devnode open time
913 : * @dev: drm_device which is being opened by userspace
914 : * @file_private: drm file-private structure to set up
915 : *
916 : * Called at device open time, sets up the structure for handling refcounting
917 : * of mm objects.
918 : */
919 : void
920 0 : drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
921 : {
922 0 : idr_init_base(&file_private->object_idr, 1);
923 0 : spin_lock_init(&file_private->table_lock);
924 0 : }
925 :
926 : /**
927 : * drm_gem_release - release file-private GEM resources
928 : * @dev: drm_device which is being closed by userspace
929 : * @file_private: drm file-private structure to clean up
930 : *
931 : * Called at close time when the filp is going away.
932 : *
933 : * Releases any remaining references on objects by this filp.
934 : */
935 : void
936 0 : drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
937 : {
938 0 : idr_for_each(&file_private->object_idr,
939 : &drm_gem_object_release_handle, file_private);
940 0 : idr_destroy(&file_private->object_idr);
941 0 : }
942 :
943 : /**
944 : * drm_gem_object_release - release GEM buffer object resources
945 : * @obj: GEM buffer object
946 : *
947 : * This releases any structures and resources used by @obj and is the inverse of
948 : * drm_gem_object_init().
949 : */
950 : void
951 0 : drm_gem_object_release(struct drm_gem_object *obj)
952 : {
953 0 : if (obj->filp)
954 0 : fput(obj->filp);
955 :
956 0 : drm_gem_private_object_fini(obj);
957 :
958 0 : drm_gem_free_mmap_offset(obj);
959 0 : drm_gem_lru_remove(obj);
960 0 : }
961 : EXPORT_SYMBOL(drm_gem_object_release);
962 :
963 : /**
964 : * drm_gem_object_free - free a GEM object
965 : * @kref: kref of the object to free
966 : *
967 : * Called after the last reference to the object has been lost.
968 : *
969 : * Frees the object
970 : */
971 : void
972 0 : drm_gem_object_free(struct kref *kref)
973 : {
974 0 : struct drm_gem_object *obj =
975 0 : container_of(kref, struct drm_gem_object, refcount);
976 :
977 0 : if (WARN_ON(!obj->funcs->free))
978 : return;
979 :
980 0 : obj->funcs->free(obj);
981 : }
982 : EXPORT_SYMBOL(drm_gem_object_free);
983 :
984 : /**
985 : * drm_gem_vm_open - vma->ops->open implementation for GEM
986 : * @vma: VM area structure
987 : *
988 : * This function implements the #vm_operations_struct open() callback for GEM
989 : * drivers. This must be used together with drm_gem_vm_close().
990 : */
991 0 : void drm_gem_vm_open(struct vm_area_struct *vma)
992 : {
993 0 : struct drm_gem_object *obj = vma->vm_private_data;
994 :
995 0 : drm_gem_object_get(obj);
996 0 : }
997 : EXPORT_SYMBOL(drm_gem_vm_open);
998 :
999 : /**
1000 : * drm_gem_vm_close - vma->ops->close implementation for GEM
1001 : * @vma: VM area structure
1002 : *
1003 : * This function implements the #vm_operations_struct close() callback for GEM
1004 : * drivers. This must be used together with drm_gem_vm_open().
1005 : */
1006 0 : void drm_gem_vm_close(struct vm_area_struct *vma)
1007 : {
1008 0 : struct drm_gem_object *obj = vma->vm_private_data;
1009 :
1010 0 : drm_gem_object_put(obj);
1011 0 : }
1012 : EXPORT_SYMBOL(drm_gem_vm_close);
1013 :
1014 : /**
1015 : * drm_gem_mmap_obj - memory map a GEM object
1016 : * @obj: the GEM object to map
1017 : * @obj_size: the object size to be mapped, in bytes
1018 : * @vma: VMA for the area to be mapped
1019 : *
1020 : * Set up the VMA to prepare mapping of the GEM object using the GEM object's
1021 : * vm_ops. Depending on their requirements, GEM objects can either
1022 : * provide a fault handler in their vm_ops (in which case any accesses to
1023 : * the object will be trapped, to perform migration, GTT binding, surface
1024 : * register allocation, or performance monitoring), or mmap the buffer memory
1025 : * synchronously after calling drm_gem_mmap_obj.
1026 : *
1027 : * This function is mainly intended to implement the DMABUF mmap operation, when
1028 : * the GEM object is not looked up based on its fake offset. To implement the
1029 : * DRM mmap operation, drivers should use the drm_gem_mmap() function.
1030 : *
1031 : * drm_gem_mmap_obj() assumes the user is granted access to the buffer while
1032 : * drm_gem_mmap() prevents unprivileged users from mapping random objects. So
1033 : * callers must verify access restrictions before calling this helper.
1034 : *
1035 : * Return 0 or success or -EINVAL if the object size is smaller than the VMA
1036 : * size, or if no vm_ops are provided.
1037 : */
1038 0 : int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
1039 : struct vm_area_struct *vma)
1040 : {
1041 : int ret;
1042 :
1043 : /* Check for valid size. */
1044 0 : if (obj_size < vma->vm_end - vma->vm_start)
1045 : return -EINVAL;
1046 :
1047 : /* Take a ref for this mapping of the object, so that the fault
1048 : * handler can dereference the mmap offset's pointer to the object.
1049 : * This reference is cleaned up by the corresponding vm_close
1050 : * (which should happen whether the vma was created by this call, or
1051 : * by a vm_open due to mremap or partial unmap or whatever).
1052 : */
1053 0 : drm_gem_object_get(obj);
1054 :
1055 0 : vma->vm_private_data = obj;
1056 0 : vma->vm_ops = obj->funcs->vm_ops;
1057 :
1058 0 : if (obj->funcs->mmap) {
1059 0 : ret = obj->funcs->mmap(obj, vma);
1060 0 : if (ret)
1061 : goto err_drm_gem_object_put;
1062 0 : WARN_ON(!(vma->vm_flags & VM_DONTEXPAND));
1063 : } else {
1064 0 : if (!vma->vm_ops) {
1065 : ret = -EINVAL;
1066 : goto err_drm_gem_object_put;
1067 : }
1068 :
1069 0 : vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1070 0 : vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1071 : vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1072 : }
1073 :
1074 : return 0;
1075 :
1076 : err_drm_gem_object_put:
1077 : drm_gem_object_put(obj);
1078 : return ret;
1079 : }
1080 : EXPORT_SYMBOL(drm_gem_mmap_obj);
1081 :
1082 : /**
1083 : * drm_gem_mmap - memory map routine for GEM objects
1084 : * @filp: DRM file pointer
1085 : * @vma: VMA for the area to be mapped
1086 : *
1087 : * If a driver supports GEM object mapping, mmap calls on the DRM file
1088 : * descriptor will end up here.
1089 : *
1090 : * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
1091 : * contain the fake offset we created when the GTT map ioctl was called on
1092 : * the object) and map it with a call to drm_gem_mmap_obj().
1093 : *
1094 : * If the caller is not granted access to the buffer object, the mmap will fail
1095 : * with EACCES. Please see the vma manager for more information.
1096 : */
1097 0 : int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1098 : {
1099 0 : struct drm_file *priv = filp->private_data;
1100 0 : struct drm_device *dev = priv->minor->dev;
1101 0 : struct drm_gem_object *obj = NULL;
1102 : struct drm_vma_offset_node *node;
1103 : int ret;
1104 :
1105 0 : if (drm_dev_is_unplugged(dev))
1106 : return -ENODEV;
1107 :
1108 0 : drm_vma_offset_lock_lookup(dev->vma_offset_manager);
1109 0 : node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
1110 : vma->vm_pgoff,
1111 : vma_pages(vma));
1112 0 : if (likely(node)) {
1113 0 : obj = container_of(node, struct drm_gem_object, vma_node);
1114 : /*
1115 : * When the object is being freed, after it hits 0-refcnt it
1116 : * proceeds to tear down the object. In the process it will
1117 : * attempt to remove the VMA offset and so acquire this
1118 : * mgr->vm_lock. Therefore if we find an object with a 0-refcnt
1119 : * that matches our range, we know it is in the process of being
1120 : * destroyed and will be freed as soon as we release the lock -
1121 : * so we have to check for the 0-refcnted object and treat it as
1122 : * invalid.
1123 : */
1124 0 : if (!kref_get_unless_zero(&obj->refcount))
1125 0 : obj = NULL;
1126 : }
1127 0 : drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
1128 :
1129 0 : if (!obj)
1130 : return -EINVAL;
1131 :
1132 0 : if (!drm_vma_node_is_allowed(node, priv)) {
1133 0 : drm_gem_object_put(obj);
1134 0 : return -EACCES;
1135 : }
1136 :
1137 0 : ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
1138 : vma);
1139 :
1140 0 : drm_gem_object_put(obj);
1141 :
1142 0 : return ret;
1143 : }
1144 : EXPORT_SYMBOL(drm_gem_mmap);
1145 :
1146 0 : void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
1147 : const struct drm_gem_object *obj)
1148 : {
1149 0 : drm_printf_indent(p, indent, "name=%d\n", obj->name);
1150 0 : drm_printf_indent(p, indent, "refcount=%u\n",
1151 : kref_read(&obj->refcount));
1152 0 : drm_printf_indent(p, indent, "start=%08lx\n",
1153 : drm_vma_node_start(&obj->vma_node));
1154 0 : drm_printf_indent(p, indent, "size=%zu\n", obj->size);
1155 0 : drm_printf_indent(p, indent, "imported=%s\n",
1156 : str_yes_no(obj->import_attach));
1157 :
1158 0 : if (obj->funcs->print_info)
1159 0 : obj->funcs->print_info(p, indent, obj);
1160 0 : }
1161 :
1162 0 : int drm_gem_pin(struct drm_gem_object *obj)
1163 : {
1164 0 : if (obj->funcs->pin)
1165 0 : return obj->funcs->pin(obj);
1166 :
1167 : return 0;
1168 : }
1169 :
1170 0 : void drm_gem_unpin(struct drm_gem_object *obj)
1171 : {
1172 0 : if (obj->funcs->unpin)
1173 0 : obj->funcs->unpin(obj);
1174 0 : }
1175 :
1176 0 : int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)
1177 : {
1178 : int ret;
1179 :
1180 : dma_resv_assert_held(obj->resv);
1181 :
1182 0 : if (!obj->funcs->vmap)
1183 : return -EOPNOTSUPP;
1184 :
1185 0 : ret = obj->funcs->vmap(obj, map);
1186 0 : if (ret)
1187 : return ret;
1188 0 : else if (iosys_map_is_null(map))
1189 : return -ENOMEM;
1190 :
1191 0 : return 0;
1192 : }
1193 : EXPORT_SYMBOL(drm_gem_vmap);
1194 :
1195 0 : void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
1196 : {
1197 : dma_resv_assert_held(obj->resv);
1198 :
1199 0 : if (iosys_map_is_null(map))
1200 : return;
1201 :
1202 0 : if (obj->funcs->vunmap)
1203 0 : obj->funcs->vunmap(obj, map);
1204 :
1205 : /* Always set the mapping to NULL. Callers may rely on this. */
1206 : iosys_map_clear(map);
1207 : }
1208 : EXPORT_SYMBOL(drm_gem_vunmap);
1209 :
1210 0 : int drm_gem_vmap_unlocked(struct drm_gem_object *obj, struct iosys_map *map)
1211 : {
1212 : int ret;
1213 :
1214 0 : dma_resv_lock(obj->resv, NULL);
1215 0 : ret = drm_gem_vmap(obj, map);
1216 0 : dma_resv_unlock(obj->resv);
1217 :
1218 0 : return ret;
1219 : }
1220 : EXPORT_SYMBOL(drm_gem_vmap_unlocked);
1221 :
1222 0 : void drm_gem_vunmap_unlocked(struct drm_gem_object *obj, struct iosys_map *map)
1223 : {
1224 0 : dma_resv_lock(obj->resv, NULL);
1225 0 : drm_gem_vunmap(obj, map);
1226 0 : dma_resv_unlock(obj->resv);
1227 0 : }
1228 : EXPORT_SYMBOL(drm_gem_vunmap_unlocked);
1229 :
1230 : /**
1231 : * drm_gem_lock_reservations - Sets up the ww context and acquires
1232 : * the lock on an array of GEM objects.
1233 : *
1234 : * Once you've locked your reservations, you'll want to set up space
1235 : * for your shared fences (if applicable), submit your job, then
1236 : * drm_gem_unlock_reservations().
1237 : *
1238 : * @objs: drm_gem_objects to lock
1239 : * @count: Number of objects in @objs
1240 : * @acquire_ctx: struct ww_acquire_ctx that will be initialized as
1241 : * part of tracking this set of locked reservations.
1242 : */
1243 : int
1244 0 : drm_gem_lock_reservations(struct drm_gem_object **objs, int count,
1245 : struct ww_acquire_ctx *acquire_ctx)
1246 : {
1247 0 : int contended = -1;
1248 : int i, ret;
1249 :
1250 : ww_acquire_init(acquire_ctx, &reservation_ww_class);
1251 :
1252 : retry:
1253 0 : if (contended != -1) {
1254 0 : struct drm_gem_object *obj = objs[contended];
1255 :
1256 0 : ret = dma_resv_lock_slow_interruptible(obj->resv,
1257 : acquire_ctx);
1258 0 : if (ret) {
1259 : ww_acquire_fini(acquire_ctx);
1260 : return ret;
1261 : }
1262 : }
1263 :
1264 0 : for (i = 0; i < count; i++) {
1265 0 : if (i == contended)
1266 0 : continue;
1267 :
1268 0 : ret = dma_resv_lock_interruptible(objs[i]->resv,
1269 : acquire_ctx);
1270 0 : if (ret) {
1271 : int j;
1272 :
1273 0 : for (j = 0; j < i; j++)
1274 0 : dma_resv_unlock(objs[j]->resv);
1275 :
1276 0 : if (contended != -1 && contended >= i)
1277 0 : dma_resv_unlock(objs[contended]->resv);
1278 :
1279 0 : if (ret == -EDEADLK) {
1280 : contended = i;
1281 : goto retry;
1282 : }
1283 :
1284 : ww_acquire_fini(acquire_ctx);
1285 : return ret;
1286 : }
1287 : }
1288 :
1289 : ww_acquire_done(acquire_ctx);
1290 :
1291 : return 0;
1292 : }
1293 : EXPORT_SYMBOL(drm_gem_lock_reservations);
1294 :
1295 : void
1296 0 : drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
1297 : struct ww_acquire_ctx *acquire_ctx)
1298 : {
1299 : int i;
1300 :
1301 0 : for (i = 0; i < count; i++)
1302 0 : dma_resv_unlock(objs[i]->resv);
1303 :
1304 0 : ww_acquire_fini(acquire_ctx);
1305 0 : }
1306 : EXPORT_SYMBOL(drm_gem_unlock_reservations);
1307 :
1308 : /**
1309 : * drm_gem_lru_init - initialize a LRU
1310 : *
1311 : * @lru: The LRU to initialize
1312 : * @lock: The lock protecting the LRU
1313 : */
1314 : void
1315 0 : drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock)
1316 : {
1317 0 : lru->lock = lock;
1318 0 : lru->count = 0;
1319 0 : INIT_LIST_HEAD(&lru->list);
1320 0 : }
1321 : EXPORT_SYMBOL(drm_gem_lru_init);
1322 :
1323 : static void
1324 0 : drm_gem_lru_remove_locked(struct drm_gem_object *obj)
1325 : {
1326 0 : obj->lru->count -= obj->size >> PAGE_SHIFT;
1327 0 : WARN_ON(obj->lru->count < 0);
1328 0 : list_del(&obj->lru_node);
1329 0 : obj->lru = NULL;
1330 0 : }
1331 :
1332 : /**
1333 : * drm_gem_lru_remove - remove object from whatever LRU it is in
1334 : *
1335 : * If the object is currently in any LRU, remove it.
1336 : *
1337 : * @obj: The GEM object to remove from current LRU
1338 : */
1339 : void
1340 0 : drm_gem_lru_remove(struct drm_gem_object *obj)
1341 : {
1342 0 : struct drm_gem_lru *lru = obj->lru;
1343 :
1344 0 : if (!lru)
1345 : return;
1346 :
1347 0 : mutex_lock(lru->lock);
1348 0 : drm_gem_lru_remove_locked(obj);
1349 0 : mutex_unlock(lru->lock);
1350 : }
1351 : EXPORT_SYMBOL(drm_gem_lru_remove);
1352 :
1353 : /**
1354 : * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU
1355 : *
1356 : * Like &drm_gem_lru_move_tail but lru lock must be held
1357 : *
1358 : * @lru: The LRU to move the object into.
1359 : * @obj: The GEM object to move into this LRU
1360 : */
1361 : void
1362 0 : drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1363 : {
1364 : lockdep_assert_held_once(lru->lock);
1365 :
1366 0 : if (obj->lru)
1367 0 : drm_gem_lru_remove_locked(obj);
1368 :
1369 0 : lru->count += obj->size >> PAGE_SHIFT;
1370 0 : list_add_tail(&obj->lru_node, &lru->list);
1371 0 : obj->lru = lru;
1372 0 : }
1373 : EXPORT_SYMBOL(drm_gem_lru_move_tail_locked);
1374 :
1375 : /**
1376 : * drm_gem_lru_move_tail - move the object to the tail of the LRU
1377 : *
1378 : * If the object is already in this LRU it will be moved to the
1379 : * tail. Otherwise it will be removed from whichever other LRU
1380 : * it is in (if any) and moved into this LRU.
1381 : *
1382 : * @lru: The LRU to move the object into.
1383 : * @obj: The GEM object to move into this LRU
1384 : */
1385 : void
1386 0 : drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1387 : {
1388 0 : mutex_lock(lru->lock);
1389 0 : drm_gem_lru_move_tail_locked(lru, obj);
1390 0 : mutex_unlock(lru->lock);
1391 0 : }
1392 : EXPORT_SYMBOL(drm_gem_lru_move_tail);
1393 :
1394 : /**
1395 : * drm_gem_lru_scan - helper to implement shrinker.scan_objects
1396 : *
1397 : * If the shrink callback succeeds, it is expected that the driver
1398 : * move the object out of this LRU.
1399 : *
1400 : * If the LRU possibly contain active buffers, it is the responsibility
1401 : * of the shrink callback to check for this (ie. dma_resv_test_signaled())
1402 : * or if necessary block until the buffer becomes idle.
1403 : *
1404 : * @lru: The LRU to scan
1405 : * @nr_to_scan: The number of pages to try to reclaim
1406 : * @remaining: The number of pages left to reclaim, should be initialized by caller
1407 : * @shrink: Callback to try to shrink/reclaim the object.
1408 : */
1409 : unsigned long
1410 0 : drm_gem_lru_scan(struct drm_gem_lru *lru,
1411 : unsigned int nr_to_scan,
1412 : unsigned long *remaining,
1413 : bool (*shrink)(struct drm_gem_object *obj))
1414 : {
1415 : struct drm_gem_lru still_in_lru;
1416 : struct drm_gem_object *obj;
1417 0 : unsigned freed = 0;
1418 :
1419 0 : drm_gem_lru_init(&still_in_lru, lru->lock);
1420 :
1421 0 : mutex_lock(lru->lock);
1422 :
1423 0 : while (freed < nr_to_scan) {
1424 0 : obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);
1425 :
1426 0 : if (!obj)
1427 : break;
1428 :
1429 0 : drm_gem_lru_move_tail_locked(&still_in_lru, obj);
1430 :
1431 : /*
1432 : * If it's in the process of being freed, gem_object->free()
1433 : * may be blocked on lock waiting to remove it. So just
1434 : * skip it.
1435 : */
1436 0 : if (!kref_get_unless_zero(&obj->refcount))
1437 0 : continue;
1438 :
1439 : /*
1440 : * Now that we own a reference, we can drop the lock for the
1441 : * rest of the loop body, to reduce contention with other
1442 : * code paths that need the LRU lock
1443 : */
1444 0 : mutex_unlock(lru->lock);
1445 :
1446 : /*
1447 : * Note that this still needs to be trylock, since we can
1448 : * hit shrinker in response to trying to get backing pages
1449 : * for this obj (ie. while it's lock is already held)
1450 : */
1451 0 : if (!dma_resv_trylock(obj->resv)) {
1452 0 : *remaining += obj->size >> PAGE_SHIFT;
1453 0 : goto tail;
1454 : }
1455 :
1456 0 : if (shrink(obj)) {
1457 0 : freed += obj->size >> PAGE_SHIFT;
1458 :
1459 : /*
1460 : * If we succeeded in releasing the object's backing
1461 : * pages, we expect the driver to have moved the object
1462 : * out of this LRU
1463 : */
1464 0 : WARN_ON(obj->lru == &still_in_lru);
1465 0 : WARN_ON(obj->lru == lru);
1466 : }
1467 :
1468 0 : dma_resv_unlock(obj->resv);
1469 :
1470 : tail:
1471 0 : drm_gem_object_put(obj);
1472 0 : mutex_lock(lru->lock);
1473 : }
1474 :
1475 : /*
1476 : * Move objects we've skipped over out of the temporary still_in_lru
1477 : * back into this LRU
1478 : */
1479 0 : list_for_each_entry (obj, &still_in_lru.list, lru_node)
1480 0 : obj->lru = lru;
1481 0 : list_splice_tail(&still_in_lru.list, &lru->list);
1482 0 : lru->count += still_in_lru.count;
1483 :
1484 0 : mutex_unlock(lru->lock);
1485 :
1486 0 : return freed;
1487 : }
1488 : EXPORT_SYMBOL(drm_gem_lru_scan);
1489 :
1490 : /**
1491 : * drm_gem_evict - helper to evict backing pages for a GEM object
1492 : * @obj: obj in question
1493 : */
1494 0 : int drm_gem_evict(struct drm_gem_object *obj)
1495 : {
1496 0 : dma_resv_assert_held(obj->resv);
1497 :
1498 0 : if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ))
1499 : return -EBUSY;
1500 :
1501 0 : if (obj->funcs->evict)
1502 0 : return obj->funcs->evict(obj);
1503 :
1504 : return 0;
1505 : }
1506 : EXPORT_SYMBOL(drm_gem_evict);
|
