Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-only
       2             : /*
       3             :  * dma-fence-util: misc functions for dma_fence objects
       4             :  *
       5             :  * Copyright (C) 2022 Advanced Micro Devices, Inc.
       6             :  * Authors:
       7             :  *      Christian König <christian.koenig@amd.com>
       8             :  */
       9             : 
      10             : #include <linux/dma-fence.h>
      11             : #include <linux/dma-fence-array.h>
      12             : #include <linux/dma-fence-chain.h>
      13             : #include <linux/dma-fence-unwrap.h>
      14             : #include <linux/slab.h>
      15             : 
      16             : /* Internal helper to start new array iteration, don't use directly */
      17             : static struct dma_fence *
      18           0 : __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
      19             : {
      20           0 :         cursor->array = dma_fence_chain_contained(cursor->chain);
      21           0 :         cursor->index = 0;
      22           0 :         return dma_fence_array_first(cursor->array);
      23             : }
      24             : 
      25             : /**
      26             :  * dma_fence_unwrap_first - return the first fence from fence containers
      27             :  * @head: the entrypoint into the containers
      28             :  * @cursor: current position inside the containers
      29             :  *
      30             :  * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
      31             :  * first fence.
      32             :  */
      33           0 : struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
      34             :                                          struct dma_fence_unwrap *cursor)
      35             : {
      36           0 :         cursor->chain = dma_fence_get(head);
      37           0 :         return __dma_fence_unwrap_array(cursor);
      38             : }
      39             : EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
      40             : 
      41             : /**
      42             :  * dma_fence_unwrap_next - return the next fence from a fence containers
      43             :  * @cursor: current position inside the containers
      44             :  *
      45             :  * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
      46             :  * the next fence from them.
      47             :  */
      48           0 : struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
      49             : {
      50             :         struct dma_fence *tmp;
      51             : 
      52           0 :         ++cursor->index;
      53           0 :         tmp = dma_fence_array_next(cursor->array, cursor->index);
      54           0 :         if (tmp)
      55             :                 return tmp;
      56             : 
      57           0 :         cursor->chain = dma_fence_chain_walk(cursor->chain);
      58           0 :         return __dma_fence_unwrap_array(cursor);
      59             : }
      60             : EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
      61             : 
      62             : /* Implementation for the dma_fence_merge() marco, don't use directly */
      63           0 : struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
      64             :                                            struct dma_fence **fences,
      65             :                                            struct dma_fence_unwrap *iter)
      66             : {
      67             :         struct dma_fence_array *result;
      68             :         struct dma_fence *tmp, **array;
      69             :         ktime_t timestamp;
      70             :         unsigned int i;
      71             :         size_t count;
      72             : 
      73           0 :         count = 0;
      74           0 :         timestamp = ns_to_ktime(0);
      75           0 :         for (i = 0; i < num_fences; ++i) {
      76           0 :                 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
      77           0 :                         if (!dma_fence_is_signaled(tmp)) {
      78           0 :                                 ++count;
      79           0 :                         } else if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT,
      80             :                                             &tmp->flags)) {
      81           0 :                                 if (ktime_after(tmp->timestamp, timestamp))
      82           0 :                                         timestamp = tmp->timestamp;
      83             :                         } else {
      84             :                                 /*
      85             :                                  * Use the current time if the fence is
      86             :                                  * currently signaling.
      87             :                                  */
      88           0 :                                 timestamp = ktime_get();
      89             :                         }
      90             :                 }
      91             :         }
      92             : 
      93             :         /*
      94             :          * If we couldn't find a pending fence just return a private signaled
      95             :          * fence with the timestamp of the last signaled one.
      96             :          */
      97           0 :         if (count == 0)
      98           0 :                 return dma_fence_allocate_private_stub(timestamp);
      99             : 
     100           0 :         array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
     101           0 :         if (!array)
     102             :                 return NULL;
     103             : 
     104             :         /*
     105             :          * This trashes the input fence array and uses it as position for the
     106             :          * following merge loop. This works because the dma_fence_merge()
     107             :          * wrapper macro is creating this temporary array on the stack together
     108             :          * with the iterators.
     109             :          */
     110           0 :         for (i = 0; i < num_fences; ++i)
     111           0 :                 fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]);
     112             : 
     113             :         count = 0;
     114             :         do {
     115             :                 unsigned int sel;
     116             : 
     117             : restart:
     118           0 :                 tmp = NULL;
     119           0 :                 for (i = 0; i < num_fences; ++i) {
     120             :                         struct dma_fence *next;
     121             : 
     122           0 :                         while (fences[i] && dma_fence_is_signaled(fences[i]))
     123           0 :                                 fences[i] = dma_fence_unwrap_next(&iter[i]);
     124             : 
     125           0 :                         next = fences[i];
     126           0 :                         if (!next)
     127           0 :                                 continue;
     128             : 
     129             :                         /*
     130             :                          * We can't guarantee that inpute fences are ordered by
     131             :                          * context, but it is still quite likely when this
     132             :                          * function is used multiple times. So attempt to order
     133             :                          * the fences by context as we pass over them and merge
     134             :                          * fences with the same context.
     135             :                          */
     136           0 :                         if (!tmp || tmp->context > next->context) {
     137             :                                 tmp = next;
     138             :                                 sel = i;
     139             : 
     140           0 :                         } else if (tmp->context < next->context) {
     141           0 :                                 continue;
     142             : 
     143           0 :                         } else if (dma_fence_is_later(tmp, next)) {
     144           0 :                                 fences[i] = dma_fence_unwrap_next(&iter[i]);
     145           0 :                                 goto restart;
     146             :                         } else {
     147           0 :                                 fences[sel] = dma_fence_unwrap_next(&iter[sel]);
     148           0 :                                 goto restart;
     149             :                         }
     150             :                 }
     151             : 
     152           0 :                 if (tmp) {
     153           0 :                         array[count++] = dma_fence_get(tmp);
     154           0 :                         fences[sel] = dma_fence_unwrap_next(&iter[sel]);
     155             :                 }
     156           0 :         } while (tmp);
     157             : 
     158           0 :         if (count == 0) {
     159           0 :                 tmp = dma_fence_allocate_private_stub(ktime_get());
     160           0 :                 goto return_tmp;
     161             :         }
     162             : 
     163           0 :         if (count == 1) {
     164           0 :                 tmp = array[0];
     165           0 :                 goto return_tmp;
     166             :         }
     167             : 
     168           0 :         result = dma_fence_array_create(count, array,
     169             :                                         dma_fence_context_alloc(1),
     170             :                                         1, false);
     171           0 :         if (!result) {
     172             :                 tmp = NULL;
     173             :                 goto return_tmp;
     174             :         }
     175           0 :         return &result->base;
     176             : 
     177             : return_tmp:
     178           0 :         kfree(array);
     179           0 :         return tmp;
     180             : }
     181             : EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);