Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : #ifndef IOU_KBUF_H
       3             : #define IOU_KBUF_H
       4             : 
       5             : #include <uapi/linux/io_uring.h>
       6             : 
       7             : struct io_buffer_list {
       8             :         /*
       9             :          * If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not,
      10             :          * then these are classic provided buffers and ->buf_list is used.
      11             :          */
      12             :         union {
      13             :                 struct list_head buf_list;
      14             :                 struct {
      15             :                         struct page **buf_pages;
      16             :                         struct io_uring_buf_ring *buf_ring;
      17             :                 };
      18             :         };
      19             :         __u16 bgid;
      20             : 
      21             :         /* below is for ring provided buffers */
      22             :         __u16 buf_nr_pages;
      23             :         __u16 nr_entries;
      24             :         __u16 head;
      25             :         __u16 mask;
      26             : 
      27             :         /* ring mapped provided buffers */
      28             :         __u8 is_mapped;
      29             :         /* ring mapped provided buffers, but mmap'ed by application */
      30             :         __u8 is_mmap;
      31             : };
      32             : 
      33             : struct io_buffer {
      34             :         struct list_head list;
      35             :         __u64 addr;
      36             :         __u32 len;
      37             :         __u16 bid;
      38             :         __u16 bgid;
      39             : };
      40             : 
      41             : void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
      42             :                               unsigned int issue_flags);
      43             : void io_destroy_buffers(struct io_ring_ctx *ctx);
      44             : 
      45             : int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
      46             : int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);
      47             : 
      48             : int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
      49             : int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);
      50             : 
      51             : int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
      52             : int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
      53             : 
      54             : unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
      55             : 
      56             : void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
      57             : 
      58             : void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid);
      59             : 
      60             : static inline void io_kbuf_recycle_ring(struct io_kiocb *req)
      61             : {
      62             :         /*
      63             :          * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
      64             :          * the flag and hence ensure that bl->head doesn't get incremented.
      65             :          * If the tail has already been incremented, hang on to it.
      66             :          * The exception is partial io, that case we should increment bl->head
      67             :          * to monopolize the buffer.
      68             :          */
      69           0 :         if (req->buf_list) {
      70           0 :                 if (req->flags & REQ_F_PARTIAL_IO) {
      71             :                         /*
      72             :                          * If we end up here, then the io_uring_lock has
      73             :                          * been kept held since we retrieved the buffer.
      74             :                          * For the io-wq case, we already cleared
      75             :                          * req->buf_list when the buffer was retrieved,
      76             :                          * hence it cannot be set here for that case.
      77             :                          */
      78           0 :                         req->buf_list->head++;
      79           0 :                         req->buf_list = NULL;
      80             :                 } else {
      81           0 :                         req->buf_index = req->buf_list->bgid;
      82           0 :                         req->flags &= ~REQ_F_BUFFER_RING;
      83             :                 }
      84             :         }
      85             : }
      86             : 
      87             : static inline bool io_do_buffer_select(struct io_kiocb *req)
      88             : {
      89           0 :         if (!(req->flags & REQ_F_BUFFER_SELECT))
      90             :                 return false;
      91           0 :         return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
      92             : }
      93             : 
      94           0 : static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
      95             : {
      96           0 :         if (req->flags & REQ_F_BUFFER_SELECTED)
      97           0 :                 io_kbuf_recycle_legacy(req, issue_flags);
      98           0 :         if (req->flags & REQ_F_BUFFER_RING)
      99             :                 io_kbuf_recycle_ring(req);
     100           0 : }
     101             : 
     102           0 : static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req,
     103             :                                               struct list_head *list)
     104             : {
     105           0 :         unsigned int ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
     106             : 
     107           0 :         if (req->flags & REQ_F_BUFFER_RING) {
     108           0 :                 if (req->buf_list) {
     109           0 :                         req->buf_index = req->buf_list->bgid;
     110           0 :                         req->buf_list->head++;
     111             :                 }
     112           0 :                 req->flags &= ~REQ_F_BUFFER_RING;
     113             :         } else {
     114           0 :                 req->buf_index = req->kbuf->bgid;
     115           0 :                 list_add(&req->kbuf->list, list);
     116           0 :                 req->flags &= ~REQ_F_BUFFER_SELECTED;
     117             :         }
     118             : 
     119           0 :         return ret;
     120             : }
     121             : 
     122             : static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)
     123             : {
     124           0 :         lockdep_assert_held(&req->ctx->completion_lock);
     125             : 
     126           0 :         if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
     127             :                 return 0;
     128           0 :         return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp);
     129             : }
     130             : 
     131             : static inline unsigned int io_put_kbuf(struct io_kiocb *req,
     132             :                                        unsigned issue_flags)
     133             : {
     134             : 
     135           0 :         if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
     136             :                 return 0;
     137           0 :         return __io_put_kbuf(req, issue_flags);
     138             : }
     139             : #endif