Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * "splice": joining two ropes together by interweaving their strands.
4 : *
5 : * This is the "extended pipe" functionality, where a pipe is used as
6 : * an arbitrary in-memory buffer. Think of a pipe as a small kernel
7 : * buffer that you can use to transfer data from one end to the other.
8 : *
9 : * The traditional unix read/write is extended with a "splice()" operation
10 : * that transfers data buffers to or from a pipe buffer.
11 : *
12 : * Named by Larry McVoy, original implementation from Linus, extended by
13 : * Jens to support splicing to files, network, direct splicing, etc and
14 : * fixing lots of bugs.
15 : *
16 : * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
17 : * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
18 : * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
19 : *
20 : */
21 : #include <linux/bvec.h>
22 : #include <linux/fs.h>
23 : #include <linux/file.h>
24 : #include <linux/pagemap.h>
25 : #include <linux/splice.h>
26 : #include <linux/memcontrol.h>
27 : #include <linux/mm_inline.h>
28 : #include <linux/swap.h>
29 : #include <linux/writeback.h>
30 : #include <linux/export.h>
31 : #include <linux/syscalls.h>
32 : #include <linux/uio.h>
33 : #include <linux/security.h>
34 : #include <linux/gfp.h>
35 : #include <linux/socket.h>
36 : #include <linux/sched/signal.h>
37 :
38 : #include "internal.h"
39 :
40 : /*
41 : * Attempt to steal a page from a pipe buffer. This should perhaps go into
42 : * a vm helper function, it's already simplified quite a bit by the
43 : * addition of remove_mapping(). If success is returned, the caller may
44 : * attempt to reuse this page for another destination.
45 : */
46 0 : static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
47 : struct pipe_buffer *buf)
48 : {
49 0 : struct folio *folio = page_folio(buf->page);
50 : struct address_space *mapping;
51 :
52 0 : folio_lock(folio);
53 :
54 0 : mapping = folio_mapping(folio);
55 0 : if (mapping) {
56 0 : WARN_ON(!folio_test_uptodate(folio));
57 :
58 : /*
59 : * At least for ext2 with nobh option, we need to wait on
60 : * writeback completing on this folio, since we'll remove it
61 : * from the pagecache. Otherwise truncate wont wait on the
62 : * folio, allowing the disk blocks to be reused by someone else
63 : * before we actually wrote our data to them. fs corruption
64 : * ensues.
65 : */
66 0 : folio_wait_writeback(folio);
67 :
68 0 : if (folio_has_private(folio) &&
69 0 : !filemap_release_folio(folio, GFP_KERNEL))
70 : goto out_unlock;
71 :
72 : /*
73 : * If we succeeded in removing the mapping, set LRU flag
74 : * and return good.
75 : */
76 0 : if (remove_mapping(mapping, folio)) {
77 0 : buf->flags |= PIPE_BUF_FLAG_LRU;
78 0 : return true;
79 : }
80 : }
81 :
82 : /*
83 : * Raced with truncate or failed to remove folio from current
84 : * address space, unlock and return failure.
85 : */
86 : out_unlock:
87 0 : folio_unlock(folio);
88 0 : return false;
89 : }
90 :
91 0 : static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
92 : struct pipe_buffer *buf)
93 : {
94 0 : put_page(buf->page);
95 0 : buf->flags &= ~PIPE_BUF_FLAG_LRU;
96 0 : }
97 :
98 : /*
99 : * Check whether the contents of buf is OK to access. Since the content
100 : * is a page cache page, IO may be in flight.
101 : */
102 0 : static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
103 : struct pipe_buffer *buf)
104 : {
105 0 : struct page *page = buf->page;
106 : int err;
107 :
108 0 : if (!PageUptodate(page)) {
109 0 : lock_page(page);
110 :
111 : /*
112 : * Page got truncated/unhashed. This will cause a 0-byte
113 : * splice, if this is the first page.
114 : */
115 0 : if (!page->mapping) {
116 : err = -ENODATA;
117 : goto error;
118 : }
119 :
120 : /*
121 : * Uh oh, read-error from disk.
122 : */
123 0 : if (!PageUptodate(page)) {
124 : err = -EIO;
125 : goto error;
126 : }
127 :
128 : /*
129 : * Page is ok afterall, we are done.
130 : */
131 0 : unlock_page(page);
132 : }
133 :
134 : return 0;
135 : error:
136 0 : unlock_page(page);
137 0 : return err;
138 : }
139 :
140 : const struct pipe_buf_operations page_cache_pipe_buf_ops = {
141 : .confirm = page_cache_pipe_buf_confirm,
142 : .release = page_cache_pipe_buf_release,
143 : .try_steal = page_cache_pipe_buf_try_steal,
144 : .get = generic_pipe_buf_get,
145 : };
146 :
147 0 : static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
148 : struct pipe_buffer *buf)
149 : {
150 0 : if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
151 : return false;
152 :
153 0 : buf->flags |= PIPE_BUF_FLAG_LRU;
154 0 : return generic_pipe_buf_try_steal(pipe, buf);
155 : }
156 :
157 : static const struct pipe_buf_operations user_page_pipe_buf_ops = {
158 : .release = page_cache_pipe_buf_release,
159 : .try_steal = user_page_pipe_buf_try_steal,
160 : .get = generic_pipe_buf_get,
161 : };
162 :
163 0 : static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
164 : {
165 0 : smp_mb();
166 0 : if (waitqueue_active(&pipe->rd_wait))
167 0 : wake_up_interruptible(&pipe->rd_wait);
168 0 : kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
169 0 : }
170 :
171 : /**
172 : * splice_to_pipe - fill passed data into a pipe
173 : * @pipe: pipe to fill
174 : * @spd: data to fill
175 : *
176 : * Description:
177 : * @spd contains a map of pages and len/offset tuples, along with
178 : * the struct pipe_buf_operations associated with these pages. This
179 : * function will link that data to the pipe.
180 : *
181 : */
182 0 : ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
183 : struct splice_pipe_desc *spd)
184 : {
185 0 : unsigned int spd_pages = spd->nr_pages;
186 0 : unsigned int tail = pipe->tail;
187 0 : unsigned int head = pipe->head;
188 0 : unsigned int mask = pipe->ring_size - 1;
189 0 : int ret = 0, page_nr = 0;
190 :
191 0 : if (!spd_pages)
192 : return 0;
193 :
194 0 : if (unlikely(!pipe->readers)) {
195 0 : send_sig(SIGPIPE, current, 0);
196 0 : ret = -EPIPE;
197 0 : goto out;
198 : }
199 :
200 0 : while (!pipe_full(head, tail, pipe->max_usage)) {
201 0 : struct pipe_buffer *buf = &pipe->bufs[head & mask];
202 :
203 0 : buf->page = spd->pages[page_nr];
204 0 : buf->offset = spd->partial[page_nr].offset;
205 0 : buf->len = spd->partial[page_nr].len;
206 0 : buf->private = spd->partial[page_nr].private;
207 0 : buf->ops = spd->ops;
208 0 : buf->flags = 0;
209 :
210 0 : head++;
211 0 : pipe->head = head;
212 0 : page_nr++;
213 0 : ret += buf->len;
214 :
215 0 : if (!--spd->nr_pages)
216 : break;
217 : }
218 :
219 0 : if (!ret)
220 0 : ret = -EAGAIN;
221 :
222 : out:
223 0 : while (page_nr < spd_pages)
224 0 : spd->spd_release(spd, page_nr++);
225 :
226 0 : return ret;
227 : }
228 : EXPORT_SYMBOL_GPL(splice_to_pipe);
229 :
230 0 : ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
231 : {
232 0 : unsigned int head = pipe->head;
233 0 : unsigned int tail = pipe->tail;
234 0 : unsigned int mask = pipe->ring_size - 1;
235 : int ret;
236 :
237 0 : if (unlikely(!pipe->readers)) {
238 0 : send_sig(SIGPIPE, current, 0);
239 0 : ret = -EPIPE;
240 0 : } else if (pipe_full(head, tail, pipe->max_usage)) {
241 : ret = -EAGAIN;
242 : } else {
243 0 : pipe->bufs[head & mask] = *buf;
244 0 : pipe->head = head + 1;
245 0 : return buf->len;
246 : }
247 0 : pipe_buf_release(pipe, buf);
248 0 : return ret;
249 : }
250 : EXPORT_SYMBOL(add_to_pipe);
251 :
252 : /*
253 : * Check if we need to grow the arrays holding pages and partial page
254 : * descriptions.
255 : */
256 0 : int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
257 : {
258 0 : unsigned int max_usage = READ_ONCE(pipe->max_usage);
259 :
260 0 : spd->nr_pages_max = max_usage;
261 0 : if (max_usage <= PIPE_DEF_BUFFERS)
262 : return 0;
263 :
264 0 : spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
265 0 : spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
266 : GFP_KERNEL);
267 :
268 0 : if (spd->pages && spd->partial)
269 : return 0;
270 :
271 0 : kfree(spd->pages);
272 0 : kfree(spd->partial);
273 0 : return -ENOMEM;
274 : }
275 :
276 0 : void splice_shrink_spd(struct splice_pipe_desc *spd)
277 : {
278 0 : if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
279 : return;
280 :
281 0 : kfree(spd->pages);
282 0 : kfree(spd->partial);
283 : }
284 :
285 : /*
286 : * Splice data from an O_DIRECT file into pages and then add them to the output
287 : * pipe.
288 : */
289 0 : ssize_t direct_splice_read(struct file *in, loff_t *ppos,
290 : struct pipe_inode_info *pipe,
291 : size_t len, unsigned int flags)
292 : {
293 : struct iov_iter to;
294 : struct bio_vec *bv;
295 : struct kiocb kiocb;
296 : struct page **pages;
297 : ssize_t ret;
298 : size_t used, npages, chunk, remain, reclaim;
299 : int i;
300 :
301 : /* Work out how much data we can actually add into the pipe */
302 0 : used = pipe_occupancy(pipe->head, pipe->tail);
303 0 : npages = max_t(ssize_t, pipe->max_usage - used, 0);
304 0 : len = min_t(size_t, len, npages * PAGE_SIZE);
305 0 : npages = DIV_ROUND_UP(len, PAGE_SIZE);
306 :
307 0 : bv = kzalloc(array_size(npages, sizeof(bv[0])) +
308 0 : array_size(npages, sizeof(struct page *)), GFP_KERNEL);
309 0 : if (!bv)
310 : return -ENOMEM;
311 :
312 0 : pages = (void *)(bv + npages);
313 0 : npages = alloc_pages_bulk_array(GFP_USER, npages, pages);
314 0 : if (!npages) {
315 0 : kfree(bv);
316 0 : return -ENOMEM;
317 : }
318 :
319 0 : remain = len = min_t(size_t, len, npages * PAGE_SIZE);
320 :
321 0 : for (i = 0; i < npages; i++) {
322 0 : chunk = min_t(size_t, PAGE_SIZE, remain);
323 0 : bv[i].bv_page = pages[i];
324 0 : bv[i].bv_offset = 0;
325 0 : bv[i].bv_len = chunk;
326 0 : remain -= chunk;
327 : }
328 :
329 : /* Do the I/O */
330 0 : iov_iter_bvec(&to, ITER_DEST, bv, npages, len);
331 0 : init_sync_kiocb(&kiocb, in);
332 0 : kiocb.ki_pos = *ppos;
333 0 : ret = call_read_iter(in, &kiocb, &to);
334 :
335 0 : reclaim = npages * PAGE_SIZE;
336 0 : remain = 0;
337 0 : if (ret > 0) {
338 0 : reclaim -= ret;
339 0 : remain = ret;
340 0 : *ppos = kiocb.ki_pos;
341 : file_accessed(in);
342 0 : } else if (ret < 0) {
343 : /*
344 : * callers of ->splice_read() expect -EAGAIN on
345 : * "can't put anything in there", rather than -EFAULT.
346 : */
347 0 : if (ret == -EFAULT)
348 0 : ret = -EAGAIN;
349 : }
350 :
351 : /* Free any pages that didn't get touched at all. */
352 0 : reclaim /= PAGE_SIZE;
353 0 : if (reclaim) {
354 0 : npages -= reclaim;
355 0 : release_pages(pages + npages, reclaim);
356 : }
357 :
358 : /* Push the remaining pages into the pipe. */
359 0 : for (i = 0; i < npages; i++) {
360 0 : struct pipe_buffer *buf = pipe_head_buf(pipe);
361 :
362 0 : chunk = min_t(size_t, remain, PAGE_SIZE);
363 0 : *buf = (struct pipe_buffer) {
364 : .ops = &default_pipe_buf_ops,
365 0 : .page = bv[i].bv_page,
366 : .offset = 0,
367 : .len = chunk,
368 : };
369 0 : pipe->head++;
370 0 : remain -= chunk;
371 : }
372 :
373 0 : kfree(bv);
374 0 : return ret;
375 : }
376 : EXPORT_SYMBOL(direct_splice_read);
377 :
378 : /**
379 : * generic_file_splice_read - splice data from file to a pipe
380 : * @in: file to splice from
381 : * @ppos: position in @in
382 : * @pipe: pipe to splice to
383 : * @len: number of bytes to splice
384 : * @flags: splice modifier flags
385 : *
386 : * Description:
387 : * Will read pages from given file and fill them into a pipe. Can be
388 : * used as long as it has more or less sane ->read_iter().
389 : *
390 : */
391 0 : ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
392 : struct pipe_inode_info *pipe, size_t len,
393 : unsigned int flags)
394 : {
395 : struct iov_iter to;
396 : struct kiocb kiocb;
397 : int ret;
398 :
399 0 : iov_iter_pipe(&to, ITER_DEST, pipe, len);
400 0 : init_sync_kiocb(&kiocb, in);
401 0 : kiocb.ki_pos = *ppos;
402 0 : ret = call_read_iter(in, &kiocb, &to);
403 0 : if (ret > 0) {
404 0 : *ppos = kiocb.ki_pos;
405 : file_accessed(in);
406 0 : } else if (ret < 0) {
407 : /* free what was emitted */
408 0 : pipe_discard_from(pipe, to.start_head);
409 : /*
410 : * callers of ->splice_read() expect -EAGAIN on
411 : * "can't put anything in there", rather than -EFAULT.
412 : */
413 0 : if (ret == -EFAULT)
414 0 : ret = -EAGAIN;
415 : }
416 :
417 0 : return ret;
418 : }
419 : EXPORT_SYMBOL(generic_file_splice_read);
420 :
421 : const struct pipe_buf_operations default_pipe_buf_ops = {
422 : .release = generic_pipe_buf_release,
423 : .try_steal = generic_pipe_buf_try_steal,
424 : .get = generic_pipe_buf_get,
425 : };
426 :
427 : /* Pipe buffer operations for a socket and similar. */
428 : const struct pipe_buf_operations nosteal_pipe_buf_ops = {
429 : .release = generic_pipe_buf_release,
430 : .get = generic_pipe_buf_get,
431 : };
432 : EXPORT_SYMBOL(nosteal_pipe_buf_ops);
433 :
434 : /*
435 : * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
436 : * using sendpage(). Return the number of bytes sent.
437 : */
438 0 : static int pipe_to_sendpage(struct pipe_inode_info *pipe,
439 : struct pipe_buffer *buf, struct splice_desc *sd)
440 : {
441 0 : struct file *file = sd->u.file;
442 0 : loff_t pos = sd->pos;
443 : int more;
444 :
445 0 : if (!likely(file->f_op->sendpage))
446 : return -EINVAL;
447 :
448 0 : more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
449 :
450 0 : if (sd->len < sd->total_len &&
451 0 : pipe_occupancy(pipe->head, pipe->tail) > 1)
452 0 : more |= MSG_SENDPAGE_NOTLAST;
453 :
454 0 : return file->f_op->sendpage(file, buf->page, buf->offset,
455 : sd->len, &pos, more);
456 : }
457 :
458 0 : static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
459 : {
460 0 : smp_mb();
461 0 : if (waitqueue_active(&pipe->wr_wait))
462 0 : wake_up_interruptible(&pipe->wr_wait);
463 0 : kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
464 0 : }
465 :
466 : /**
467 : * splice_from_pipe_feed - feed available data from a pipe to a file
468 : * @pipe: pipe to splice from
469 : * @sd: information to @actor
470 : * @actor: handler that splices the data
471 : *
472 : * Description:
473 : * This function loops over the pipe and calls @actor to do the
474 : * actual moving of a single struct pipe_buffer to the desired
475 : * destination. It returns when there's no more buffers left in
476 : * the pipe or if the requested number of bytes (@sd->total_len)
477 : * have been copied. It returns a positive number (one) if the
478 : * pipe needs to be filled with more data, zero if the required
479 : * number of bytes have been copied and -errno on error.
480 : *
481 : * This, together with splice_from_pipe_{begin,end,next}, may be
482 : * used to implement the functionality of __splice_from_pipe() when
483 : * locking is required around copying the pipe buffers to the
484 : * destination.
485 : */
486 0 : static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
487 : splice_actor *actor)
488 : {
489 0 : unsigned int head = pipe->head;
490 0 : unsigned int tail = pipe->tail;
491 0 : unsigned int mask = pipe->ring_size - 1;
492 : int ret;
493 :
494 0 : while (!pipe_empty(head, tail)) {
495 0 : struct pipe_buffer *buf = &pipe->bufs[tail & mask];
496 :
497 0 : sd->len = buf->len;
498 0 : if (sd->len > sd->total_len)
499 0 : sd->len = sd->total_len;
500 :
501 0 : ret = pipe_buf_confirm(pipe, buf);
502 0 : if (unlikely(ret)) {
503 0 : if (ret == -ENODATA)
504 0 : ret = 0;
505 : return ret;
506 : }
507 :
508 0 : ret = actor(pipe, buf, sd);
509 0 : if (ret <= 0)
510 : return ret;
511 :
512 0 : buf->offset += ret;
513 0 : buf->len -= ret;
514 :
515 0 : sd->num_spliced += ret;
516 0 : sd->len -= ret;
517 0 : sd->pos += ret;
518 0 : sd->total_len -= ret;
519 :
520 0 : if (!buf->len) {
521 0 : pipe_buf_release(pipe, buf);
522 0 : tail++;
523 0 : pipe->tail = tail;
524 0 : if (pipe->files)
525 0 : sd->need_wakeup = true;
526 : }
527 :
528 0 : if (!sd->total_len)
529 : return 0;
530 : }
531 :
532 : return 1;
533 : }
534 :
535 : /* We know we have a pipe buffer, but maybe it's empty? */
536 0 : static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
537 : {
538 0 : unsigned int tail = pipe->tail;
539 0 : unsigned int mask = pipe->ring_size - 1;
540 0 : struct pipe_buffer *buf = &pipe->bufs[tail & mask];
541 :
542 0 : if (unlikely(!buf->len)) {
543 0 : pipe_buf_release(pipe, buf);
544 0 : pipe->tail = tail+1;
545 0 : return true;
546 : }
547 :
548 : return false;
549 : }
550 :
551 : /**
552 : * splice_from_pipe_next - wait for some data to splice from
553 : * @pipe: pipe to splice from
554 : * @sd: information about the splice operation
555 : *
556 : * Description:
557 : * This function will wait for some data and return a positive
558 : * value (one) if pipe buffers are available. It will return zero
559 : * or -errno if no more data needs to be spliced.
560 : */
561 0 : static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
562 : {
563 : /*
564 : * Check for signal early to make process killable when there are
565 : * always buffers available
566 : */
567 0 : if (signal_pending(current))
568 : return -ERESTARTSYS;
569 :
570 : repeat:
571 0 : while (pipe_empty(pipe->head, pipe->tail)) {
572 0 : if (!pipe->writers)
573 : return 0;
574 :
575 0 : if (sd->num_spliced)
576 : return 0;
577 :
578 0 : if (sd->flags & SPLICE_F_NONBLOCK)
579 : return -EAGAIN;
580 :
581 0 : if (signal_pending(current))
582 : return -ERESTARTSYS;
583 :
584 0 : if (sd->need_wakeup) {
585 0 : wakeup_pipe_writers(pipe);
586 0 : sd->need_wakeup = false;
587 : }
588 :
589 0 : pipe_wait_readable(pipe);
590 : }
591 :
592 0 : if (eat_empty_buffer(pipe))
593 : goto repeat;
594 :
595 : return 1;
596 : }
597 :
598 : /**
599 : * splice_from_pipe_begin - start splicing from pipe
600 : * @sd: information about the splice operation
601 : *
602 : * Description:
603 : * This function should be called before a loop containing
604 : * splice_from_pipe_next() and splice_from_pipe_feed() to
605 : * initialize the necessary fields of @sd.
606 : */
607 : static void splice_from_pipe_begin(struct splice_desc *sd)
608 : {
609 0 : sd->num_spliced = 0;
610 0 : sd->need_wakeup = false;
611 : }
612 :
613 : /**
614 : * splice_from_pipe_end - finish splicing from pipe
615 : * @pipe: pipe to splice from
616 : * @sd: information about the splice operation
617 : *
618 : * Description:
619 : * This function will wake up pipe writers if necessary. It should
620 : * be called after a loop containing splice_from_pipe_next() and
621 : * splice_from_pipe_feed().
622 : */
623 : static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
624 : {
625 0 : if (sd->need_wakeup)
626 0 : wakeup_pipe_writers(pipe);
627 : }
628 :
629 : /**
630 : * __splice_from_pipe - splice data from a pipe to given actor
631 : * @pipe: pipe to splice from
632 : * @sd: information to @actor
633 : * @actor: handler that splices the data
634 : *
635 : * Description:
636 : * This function does little more than loop over the pipe and call
637 : * @actor to do the actual moving of a single struct pipe_buffer to
638 : * the desired destination. See pipe_to_file, pipe_to_sendpage, or
639 : * pipe_to_user.
640 : *
641 : */
642 0 : ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
643 : splice_actor *actor)
644 : {
645 : int ret;
646 :
647 0 : splice_from_pipe_begin(sd);
648 : do {
649 0 : cond_resched();
650 0 : ret = splice_from_pipe_next(pipe, sd);
651 0 : if (ret > 0)
652 0 : ret = splice_from_pipe_feed(pipe, sd, actor);
653 0 : } while (ret > 0);
654 0 : splice_from_pipe_end(pipe, sd);
655 :
656 0 : return sd->num_spliced ? sd->num_spliced : ret;
657 : }
658 : EXPORT_SYMBOL(__splice_from_pipe);
659 :
660 : /**
661 : * splice_from_pipe - splice data from a pipe to a file
662 : * @pipe: pipe to splice from
663 : * @out: file to splice to
664 : * @ppos: position in @out
665 : * @len: how many bytes to splice
666 : * @flags: splice modifier flags
667 : * @actor: handler that splices the data
668 : *
669 : * Description:
670 : * See __splice_from_pipe. This function locks the pipe inode,
671 : * otherwise it's identical to __splice_from_pipe().
672 : *
673 : */
674 0 : ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
675 : loff_t *ppos, size_t len, unsigned int flags,
676 : splice_actor *actor)
677 : {
678 : ssize_t ret;
679 0 : struct splice_desc sd = {
680 : .total_len = len,
681 : .flags = flags,
682 0 : .pos = *ppos,
683 : .u.file = out,
684 : };
685 :
686 0 : pipe_lock(pipe);
687 0 : ret = __splice_from_pipe(pipe, &sd, actor);
688 0 : pipe_unlock(pipe);
689 :
690 0 : return ret;
691 : }
692 :
693 : /**
694 : * iter_file_splice_write - splice data from a pipe to a file
695 : * @pipe: pipe info
696 : * @out: file to write to
697 : * @ppos: position in @out
698 : * @len: number of bytes to splice
699 : * @flags: splice modifier flags
700 : *
701 : * Description:
702 : * Will either move or copy pages (determined by @flags options) from
703 : * the given pipe inode to the given file.
704 : * This one is ->write_iter-based.
705 : *
706 : */
707 : ssize_t
708 0 : iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
709 : loff_t *ppos, size_t len, unsigned int flags)
710 : {
711 0 : struct splice_desc sd = {
712 : .total_len = len,
713 : .flags = flags,
714 0 : .pos = *ppos,
715 : .u.file = out,
716 : };
717 0 : int nbufs = pipe->max_usage;
718 0 : struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
719 : GFP_KERNEL);
720 : ssize_t ret;
721 :
722 0 : if (unlikely(!array))
723 : return -ENOMEM;
724 :
725 0 : pipe_lock(pipe);
726 :
727 0 : splice_from_pipe_begin(&sd);
728 0 : while (sd.total_len) {
729 : struct iov_iter from;
730 : unsigned int head, tail, mask;
731 : size_t left;
732 : int n;
733 :
734 0 : ret = splice_from_pipe_next(pipe, &sd);
735 0 : if (ret <= 0)
736 : break;
737 :
738 0 : if (unlikely(nbufs < pipe->max_usage)) {
739 0 : kfree(array);
740 0 : nbufs = pipe->max_usage;
741 0 : array = kcalloc(nbufs, sizeof(struct bio_vec),
742 : GFP_KERNEL);
743 0 : if (!array) {
744 : ret = -ENOMEM;
745 : break;
746 : }
747 : }
748 :
749 0 : head = pipe->head;
750 0 : tail = pipe->tail;
751 0 : mask = pipe->ring_size - 1;
752 :
753 : /* build the vector */
754 0 : left = sd.total_len;
755 0 : for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
756 0 : struct pipe_buffer *buf = &pipe->bufs[tail & mask];
757 0 : size_t this_len = buf->len;
758 :
759 : /* zero-length bvecs are not supported, skip them */
760 0 : if (!this_len)
761 0 : continue;
762 0 : this_len = min(this_len, left);
763 :
764 0 : ret = pipe_buf_confirm(pipe, buf);
765 0 : if (unlikely(ret)) {
766 0 : if (ret == -ENODATA)
767 0 : ret = 0;
768 0 : goto done;
769 : }
770 :
771 0 : bvec_set_page(&array[n], buf->page, this_len,
772 : buf->offset);
773 0 : left -= this_len;
774 0 : n++;
775 : }
776 :
777 0 : iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left);
778 0 : ret = vfs_iter_write(out, &from, &sd.pos, 0);
779 0 : if (ret <= 0)
780 : break;
781 :
782 0 : sd.num_spliced += ret;
783 0 : sd.total_len -= ret;
784 0 : *ppos = sd.pos;
785 :
786 : /* dismiss the fully eaten buffers, adjust the partial one */
787 0 : tail = pipe->tail;
788 0 : while (ret) {
789 0 : struct pipe_buffer *buf = &pipe->bufs[tail & mask];
790 0 : if (ret >= buf->len) {
791 0 : ret -= buf->len;
792 0 : buf->len = 0;
793 0 : pipe_buf_release(pipe, buf);
794 0 : tail++;
795 0 : pipe->tail = tail;
796 0 : if (pipe->files)
797 0 : sd.need_wakeup = true;
798 : } else {
799 0 : buf->offset += ret;
800 0 : buf->len -= ret;
801 0 : ret = 0;
802 : }
803 : }
804 : }
805 : done:
806 0 : kfree(array);
807 0 : splice_from_pipe_end(pipe, &sd);
808 :
809 0 : pipe_unlock(pipe);
810 :
811 0 : if (sd.num_spliced)
812 0 : ret = sd.num_spliced;
813 :
814 : return ret;
815 : }
816 :
817 : EXPORT_SYMBOL(iter_file_splice_write);
818 :
819 : /**
820 : * generic_splice_sendpage - splice data from a pipe to a socket
821 : * @pipe: pipe to splice from
822 : * @out: socket to write to
823 : * @ppos: position in @out
824 : * @len: number of bytes to splice
825 : * @flags: splice modifier flags
826 : *
827 : * Description:
828 : * Will send @len bytes from the pipe to a network socket. No data copying
829 : * is involved.
830 : *
831 : */
832 0 : ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
833 : loff_t *ppos, size_t len, unsigned int flags)
834 : {
835 0 : return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
836 : }
837 :
838 : EXPORT_SYMBOL(generic_splice_sendpage);
839 :
840 : static int warn_unsupported(struct file *file, const char *op)
841 : {
842 : pr_debug_ratelimited(
843 : "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
844 : op, file, current->pid, current->comm);
845 : return -EINVAL;
846 : }
847 :
848 : /*
849 : * Attempt to initiate a splice from pipe to file.
850 : */
851 : static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
852 : loff_t *ppos, size_t len, unsigned int flags)
853 : {
854 0 : if (unlikely(!out->f_op->splice_write))
855 : return warn_unsupported(out, "write");
856 0 : return out->f_op->splice_write(pipe, out, ppos, len, flags);
857 : }
858 :
859 : /*
860 : * Attempt to initiate a splice from a file to a pipe.
861 : */
862 0 : static long do_splice_to(struct file *in, loff_t *ppos,
863 : struct pipe_inode_info *pipe, size_t len,
864 : unsigned int flags)
865 : {
866 : unsigned int p_space;
867 : int ret;
868 :
869 0 : if (unlikely(!(in->f_mode & FMODE_READ)))
870 : return -EBADF;
871 :
872 : /* Don't try to read more the pipe has space for. */
873 0 : p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);
874 0 : len = min_t(size_t, len, p_space << PAGE_SHIFT);
875 :
876 0 : ret = rw_verify_area(READ, in, ppos, len);
877 0 : if (unlikely(ret < 0))
878 0 : return ret;
879 :
880 0 : if (unlikely(len > MAX_RW_COUNT))
881 0 : len = MAX_RW_COUNT;
882 :
883 0 : if (unlikely(!in->f_op->splice_read))
884 : return warn_unsupported(in, "read");
885 0 : return in->f_op->splice_read(in, ppos, pipe, len, flags);
886 : }
887 :
888 : /**
889 : * splice_direct_to_actor - splices data directly between two non-pipes
890 : * @in: file to splice from
891 : * @sd: actor information on where to splice to
892 : * @actor: handles the data splicing
893 : *
894 : * Description:
895 : * This is a special case helper to splice directly between two
896 : * points, without requiring an explicit pipe. Internally an allocated
897 : * pipe is cached in the process, and reused during the lifetime of
898 : * that process.
899 : *
900 : */
901 0 : ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
902 : splice_direct_actor *actor)
903 : {
904 : struct pipe_inode_info *pipe;
905 : long ret, bytes;
906 : size_t len;
907 : int i, flags, more;
908 :
909 : /*
910 : * We require the input to be seekable, as we don't want to randomly
911 : * drop data for eg socket -> socket splicing. Use the piped splicing
912 : * for that!
913 : */
914 0 : if (unlikely(!(in->f_mode & FMODE_LSEEK)))
915 : return -EINVAL;
916 :
917 : /*
918 : * neither in nor out is a pipe, setup an internal pipe attached to
919 : * 'out' and transfer the wanted data from 'in' to 'out' through that
920 : */
921 0 : pipe = current->splice_pipe;
922 0 : if (unlikely(!pipe)) {
923 0 : pipe = alloc_pipe_info();
924 0 : if (!pipe)
925 : return -ENOMEM;
926 :
927 : /*
928 : * We don't have an immediate reader, but we'll read the stuff
929 : * out of the pipe right after the splice_to_pipe(). So set
930 : * PIPE_READERS appropriately.
931 : */
932 0 : pipe->readers = 1;
933 :
934 0 : current->splice_pipe = pipe;
935 : }
936 :
937 : /*
938 : * Do the splice.
939 : */
940 0 : ret = 0;
941 0 : bytes = 0;
942 0 : len = sd->total_len;
943 0 : flags = sd->flags;
944 :
945 : /*
946 : * Don't block on output, we have to drain the direct pipe.
947 : */
948 0 : sd->flags &= ~SPLICE_F_NONBLOCK;
949 0 : more = sd->flags & SPLICE_F_MORE;
950 :
951 0 : WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
952 :
953 0 : while (len) {
954 : size_t read_len;
955 0 : loff_t pos = sd->pos, prev_pos = pos;
956 :
957 0 : ret = do_splice_to(in, &pos, pipe, len, flags);
958 0 : if (unlikely(ret <= 0))
959 : goto out_release;
960 :
961 0 : read_len = ret;
962 0 : sd->total_len = read_len;
963 :
964 : /*
965 : * If more data is pending, set SPLICE_F_MORE
966 : * If this is the last data and SPLICE_F_MORE was not set
967 : * initially, clears it.
968 : */
969 0 : if (read_len < len)
970 0 : sd->flags |= SPLICE_F_MORE;
971 0 : else if (!more)
972 0 : sd->flags &= ~SPLICE_F_MORE;
973 : /*
974 : * NOTE: nonblocking mode only applies to the input. We
975 : * must not do the output in nonblocking mode as then we
976 : * could get stuck data in the internal pipe:
977 : */
978 0 : ret = actor(pipe, sd);
979 0 : if (unlikely(ret <= 0)) {
980 0 : sd->pos = prev_pos;
981 0 : goto out_release;
982 : }
983 :
984 0 : bytes += ret;
985 0 : len -= ret;
986 0 : sd->pos = pos;
987 :
988 0 : if (ret < read_len) {
989 0 : sd->pos = prev_pos + ret;
990 0 : goto out_release;
991 : }
992 : }
993 :
994 : done:
995 0 : pipe->tail = pipe->head = 0;
996 : file_accessed(in);
997 : return bytes;
998 :
999 : out_release:
1000 : /*
1001 : * If we did an incomplete transfer we must release
1002 : * the pipe buffers in question:
1003 : */
1004 0 : for (i = 0; i < pipe->ring_size; i++) {
1005 0 : struct pipe_buffer *buf = &pipe->bufs[i];
1006 :
1007 0 : if (buf->ops)
1008 : pipe_buf_release(pipe, buf);
1009 : }
1010 :
1011 0 : if (!bytes)
1012 0 : bytes = ret;
1013 :
1014 : goto done;
1015 : }
1016 : EXPORT_SYMBOL(splice_direct_to_actor);
1017 :
1018 0 : static int direct_splice_actor(struct pipe_inode_info *pipe,
1019 : struct splice_desc *sd)
1020 : {
1021 0 : struct file *file = sd->u.file;
1022 :
1023 0 : return do_splice_from(pipe, file, sd->opos, sd->total_len,
1024 : sd->flags);
1025 : }
1026 :
1027 : /**
1028 : * do_splice_direct - splices data directly between two files
1029 : * @in: file to splice from
1030 : * @ppos: input file offset
1031 : * @out: file to splice to
1032 : * @opos: output file offset
1033 : * @len: number of bytes to splice
1034 : * @flags: splice modifier flags
1035 : *
1036 : * Description:
1037 : * For use by do_sendfile(). splice can easily emulate sendfile, but
1038 : * doing it in the application would incur an extra system call
1039 : * (splice in + splice out, as compared to just sendfile()). So this helper
1040 : * can splice directly through a process-private pipe.
1041 : *
1042 : */
1043 0 : long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1044 : loff_t *opos, size_t len, unsigned int flags)
1045 : {
1046 0 : struct splice_desc sd = {
1047 : .len = len,
1048 : .total_len = len,
1049 : .flags = flags,
1050 0 : .pos = *ppos,
1051 : .u.file = out,
1052 : .opos = opos,
1053 : };
1054 : long ret;
1055 :
1056 0 : if (unlikely(!(out->f_mode & FMODE_WRITE)))
1057 : return -EBADF;
1058 :
1059 0 : if (unlikely(out->f_flags & O_APPEND))
1060 : return -EINVAL;
1061 :
1062 0 : ret = rw_verify_area(WRITE, out, opos, len);
1063 0 : if (unlikely(ret < 0))
1064 : return ret;
1065 :
1066 0 : ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1067 0 : if (ret > 0)
1068 0 : *ppos = sd.pos;
1069 :
1070 : return ret;
1071 : }
1072 : EXPORT_SYMBOL(do_splice_direct);
1073 :
1074 0 : static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
1075 : {
1076 : for (;;) {
1077 0 : if (unlikely(!pipe->readers)) {
1078 0 : send_sig(SIGPIPE, current, 0);
1079 0 : return -EPIPE;
1080 : }
1081 0 : if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1082 : return 0;
1083 0 : if (flags & SPLICE_F_NONBLOCK)
1084 : return -EAGAIN;
1085 0 : if (signal_pending(current))
1086 : return -ERESTARTSYS;
1087 0 : pipe_wait_writable(pipe);
1088 : }
1089 : }
1090 :
1091 : static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1092 : struct pipe_inode_info *opipe,
1093 : size_t len, unsigned int flags);
1094 :
1095 0 : long splice_file_to_pipe(struct file *in,
1096 : struct pipe_inode_info *opipe,
1097 : loff_t *offset,
1098 : size_t len, unsigned int flags)
1099 : {
1100 : long ret;
1101 :
1102 0 : pipe_lock(opipe);
1103 0 : ret = wait_for_space(opipe, flags);
1104 0 : if (!ret)
1105 0 : ret = do_splice_to(in, offset, opipe, len, flags);
1106 0 : pipe_unlock(opipe);
1107 0 : if (ret > 0)
1108 0 : wakeup_pipe_readers(opipe);
1109 0 : return ret;
1110 : }
1111 :
1112 : /*
1113 : * Determine where to splice to/from.
1114 : */
1115 0 : long do_splice(struct file *in, loff_t *off_in, struct file *out,
1116 : loff_t *off_out, size_t len, unsigned int flags)
1117 : {
1118 : struct pipe_inode_info *ipipe;
1119 : struct pipe_inode_info *opipe;
1120 : loff_t offset;
1121 : long ret;
1122 :
1123 0 : if (unlikely(!(in->f_mode & FMODE_READ) ||
1124 : !(out->f_mode & FMODE_WRITE)))
1125 : return -EBADF;
1126 :
1127 0 : ipipe = get_pipe_info(in, true);
1128 0 : opipe = get_pipe_info(out, true);
1129 :
1130 0 : if (ipipe && opipe) {
1131 0 : if (off_in || off_out)
1132 : return -ESPIPE;
1133 :
1134 : /* Splicing to self would be fun, but... */
1135 0 : if (ipipe == opipe)
1136 : return -EINVAL;
1137 :
1138 0 : if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1139 0 : flags |= SPLICE_F_NONBLOCK;
1140 :
1141 0 : return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1142 : }
1143 :
1144 0 : if (ipipe) {
1145 0 : if (off_in)
1146 : return -ESPIPE;
1147 0 : if (off_out) {
1148 0 : if (!(out->f_mode & FMODE_PWRITE))
1149 : return -EINVAL;
1150 0 : offset = *off_out;
1151 : } else {
1152 0 : offset = out->f_pos;
1153 : }
1154 :
1155 0 : if (unlikely(out->f_flags & O_APPEND))
1156 : return -EINVAL;
1157 :
1158 0 : ret = rw_verify_area(WRITE, out, &offset, len);
1159 0 : if (unlikely(ret < 0))
1160 : return ret;
1161 :
1162 0 : if (in->f_flags & O_NONBLOCK)
1163 0 : flags |= SPLICE_F_NONBLOCK;
1164 :
1165 0 : file_start_write(out);
1166 0 : ret = do_splice_from(ipipe, out, &offset, len, flags);
1167 0 : file_end_write(out);
1168 :
1169 0 : if (!off_out)
1170 0 : out->f_pos = offset;
1171 : else
1172 0 : *off_out = offset;
1173 :
1174 : return ret;
1175 : }
1176 :
1177 0 : if (opipe) {
1178 0 : if (off_out)
1179 : return -ESPIPE;
1180 0 : if (off_in) {
1181 0 : if (!(in->f_mode & FMODE_PREAD))
1182 : return -EINVAL;
1183 0 : offset = *off_in;
1184 : } else {
1185 0 : offset = in->f_pos;
1186 : }
1187 :
1188 0 : if (out->f_flags & O_NONBLOCK)
1189 0 : flags |= SPLICE_F_NONBLOCK;
1190 :
1191 0 : ret = splice_file_to_pipe(in, opipe, &offset, len, flags);
1192 0 : if (!off_in)
1193 0 : in->f_pos = offset;
1194 : else
1195 0 : *off_in = offset;
1196 :
1197 : return ret;
1198 : }
1199 :
1200 : return -EINVAL;
1201 : }
1202 :
1203 0 : static long __do_splice(struct file *in, loff_t __user *off_in,
1204 : struct file *out, loff_t __user *off_out,
1205 : size_t len, unsigned int flags)
1206 : {
1207 : struct pipe_inode_info *ipipe;
1208 : struct pipe_inode_info *opipe;
1209 0 : loff_t offset, *__off_in = NULL, *__off_out = NULL;
1210 : long ret;
1211 :
1212 0 : ipipe = get_pipe_info(in, true);
1213 0 : opipe = get_pipe_info(out, true);
1214 :
1215 0 : if (ipipe && off_in)
1216 : return -ESPIPE;
1217 0 : if (opipe && off_out)
1218 : return -ESPIPE;
1219 :
1220 0 : if (off_out) {
1221 0 : if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1222 : return -EFAULT;
1223 : __off_out = &offset;
1224 : }
1225 0 : if (off_in) {
1226 0 : if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1227 : return -EFAULT;
1228 : __off_in = &offset;
1229 : }
1230 :
1231 0 : ret = do_splice(in, __off_in, out, __off_out, len, flags);
1232 0 : if (ret < 0)
1233 : return ret;
1234 :
1235 0 : if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
1236 : return -EFAULT;
1237 0 : if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
1238 : return -EFAULT;
1239 :
1240 : return ret;
1241 : }
1242 :
1243 0 : static int iter_to_pipe(struct iov_iter *from,
1244 : struct pipe_inode_info *pipe,
1245 : unsigned flags)
1246 : {
1247 0 : struct pipe_buffer buf = {
1248 : .ops = &user_page_pipe_buf_ops,
1249 : .flags = flags
1250 : };
1251 0 : size_t total = 0;
1252 0 : int ret = 0;
1253 :
1254 0 : while (iov_iter_count(from)) {
1255 : struct page *pages[16];
1256 : ssize_t left;
1257 : size_t start;
1258 : int i, n;
1259 :
1260 0 : left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start);
1261 0 : if (left <= 0) {
1262 0 : ret = left;
1263 0 : break;
1264 : }
1265 :
1266 0 : n = DIV_ROUND_UP(left + start, PAGE_SIZE);
1267 0 : for (i = 0; i < n; i++) {
1268 0 : int size = min_t(int, left, PAGE_SIZE - start);
1269 :
1270 0 : buf.page = pages[i];
1271 0 : buf.offset = start;
1272 0 : buf.len = size;
1273 0 : ret = add_to_pipe(pipe, &buf);
1274 0 : if (unlikely(ret < 0)) {
1275 0 : iov_iter_revert(from, left);
1276 : // this one got dropped by add_to_pipe()
1277 0 : while (++i < n)
1278 0 : put_page(pages[i]);
1279 0 : goto out;
1280 : }
1281 0 : total += ret;
1282 0 : left -= size;
1283 0 : start = 0;
1284 : }
1285 : }
1286 : out:
1287 0 : return total ? total : ret;
1288 : }
1289 :
1290 0 : static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1291 : struct splice_desc *sd)
1292 : {
1293 0 : int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
1294 0 : return n == sd->len ? n : -EFAULT;
1295 : }
1296 :
1297 : /*
1298 : * For lack of a better implementation, implement vmsplice() to userspace
1299 : * as a simple copy of the pipes pages to the user iov.
1300 : */
1301 0 : static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
1302 : unsigned int flags)
1303 : {
1304 0 : struct pipe_inode_info *pipe = get_pipe_info(file, true);
1305 0 : struct splice_desc sd = {
1306 0 : .total_len = iov_iter_count(iter),
1307 : .flags = flags,
1308 : .u.data = iter
1309 : };
1310 0 : long ret = 0;
1311 :
1312 0 : if (!pipe)
1313 : return -EBADF;
1314 :
1315 0 : if (sd.total_len) {
1316 0 : pipe_lock(pipe);
1317 0 : ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1318 0 : pipe_unlock(pipe);
1319 : }
1320 :
1321 : return ret;
1322 : }
1323 :
1324 : /*
1325 : * vmsplice splices a user address range into a pipe. It can be thought of
1326 : * as splice-from-memory, where the regular splice is splice-from-file (or
1327 : * to file). In both cases the output is a pipe, naturally.
1328 : */
1329 0 : static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
1330 : unsigned int flags)
1331 : {
1332 : struct pipe_inode_info *pipe;
1333 0 : long ret = 0;
1334 0 : unsigned buf_flag = 0;
1335 :
1336 0 : if (flags & SPLICE_F_GIFT)
1337 0 : buf_flag = PIPE_BUF_FLAG_GIFT;
1338 :
1339 0 : pipe = get_pipe_info(file, true);
1340 0 : if (!pipe)
1341 : return -EBADF;
1342 :
1343 0 : pipe_lock(pipe);
1344 0 : ret = wait_for_space(pipe, flags);
1345 0 : if (!ret)
1346 0 : ret = iter_to_pipe(iter, pipe, buf_flag);
1347 0 : pipe_unlock(pipe);
1348 0 : if (ret > 0)
1349 0 : wakeup_pipe_readers(pipe);
1350 : return ret;
1351 : }
1352 :
1353 0 : static int vmsplice_type(struct fd f, int *type)
1354 : {
1355 0 : if (!f.file)
1356 : return -EBADF;
1357 0 : if (f.file->f_mode & FMODE_WRITE) {
1358 0 : *type = ITER_SOURCE;
1359 0 : } else if (f.file->f_mode & FMODE_READ) {
1360 0 : *type = ITER_DEST;
1361 : } else {
1362 0 : fdput(f);
1363 : return -EBADF;
1364 : }
1365 : return 0;
1366 : }
1367 :
1368 : /*
1369 : * Note that vmsplice only really supports true splicing _from_ user memory
1370 : * to a pipe, not the other way around. Splicing from user memory is a simple
1371 : * operation that can be supported without any funky alignment restrictions
1372 : * or nasty vm tricks. We simply map in the user memory and fill them into
1373 : * a pipe. The reverse isn't quite as easy, though. There are two possible
1374 : * solutions for that:
1375 : *
1376 : * - memcpy() the data internally, at which point we might as well just
1377 : * do a regular read() on the buffer anyway.
1378 : * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1379 : * has restriction limitations on both ends of the pipe).
1380 : *
1381 : * Currently we punt and implement it as a normal copy, see pipe_to_user().
1382 : *
1383 : */
1384 0 : SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
1385 : unsigned long, nr_segs, unsigned int, flags)
1386 : {
1387 : struct iovec iovstack[UIO_FASTIOV];
1388 0 : struct iovec *iov = iovstack;
1389 : struct iov_iter iter;
1390 : ssize_t error;
1391 : struct fd f;
1392 : int type;
1393 :
1394 0 : if (unlikely(flags & ~SPLICE_F_ALL))
1395 : return -EINVAL;
1396 :
1397 0 : f = fdget(fd);
1398 0 : error = vmsplice_type(f, &type);
1399 0 : if (error)
1400 : return error;
1401 :
1402 0 : error = import_iovec(type, uiov, nr_segs,
1403 : ARRAY_SIZE(iovstack), &iov, &iter);
1404 0 : if (error < 0)
1405 : goto out_fdput;
1406 :
1407 0 : if (!iov_iter_count(&iter))
1408 : error = 0;
1409 0 : else if (type == ITER_SOURCE)
1410 0 : error = vmsplice_to_pipe(f.file, &iter, flags);
1411 : else
1412 0 : error = vmsplice_to_user(f.file, &iter, flags);
1413 :
1414 0 : kfree(iov);
1415 : out_fdput:
1416 0 : fdput(f);
1417 : return error;
1418 : }
1419 :
1420 0 : SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1421 : int, fd_out, loff_t __user *, off_out,
1422 : size_t, len, unsigned int, flags)
1423 : {
1424 : struct fd in, out;
1425 : long error;
1426 :
1427 0 : if (unlikely(!len))
1428 : return 0;
1429 :
1430 0 : if (unlikely(flags & ~SPLICE_F_ALL))
1431 : return -EINVAL;
1432 :
1433 0 : error = -EBADF;
1434 0 : in = fdget(fd_in);
1435 0 : if (in.file) {
1436 0 : out = fdget(fd_out);
1437 0 : if (out.file) {
1438 0 : error = __do_splice(in.file, off_in, out.file, off_out,
1439 : len, flags);
1440 0 : fdput(out);
1441 : }
1442 0 : fdput(in);
1443 : }
1444 : return error;
1445 : }
1446 :
1447 : /*
1448 : * Make sure there's data to read. Wait for input if we can, otherwise
1449 : * return an appropriate error.
1450 : */
1451 0 : static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1452 : {
1453 : int ret;
1454 :
1455 : /*
1456 : * Check the pipe occupancy without the inode lock first. This function
1457 : * is speculative anyways, so missing one is ok.
1458 : */
1459 0 : if (!pipe_empty(pipe->head, pipe->tail))
1460 : return 0;
1461 :
1462 0 : ret = 0;
1463 0 : pipe_lock(pipe);
1464 :
1465 0 : while (pipe_empty(pipe->head, pipe->tail)) {
1466 0 : if (signal_pending(current)) {
1467 : ret = -ERESTARTSYS;
1468 : break;
1469 : }
1470 0 : if (!pipe->writers)
1471 : break;
1472 0 : if (flags & SPLICE_F_NONBLOCK) {
1473 : ret = -EAGAIN;
1474 : break;
1475 : }
1476 0 : pipe_wait_readable(pipe);
1477 : }
1478 :
1479 0 : pipe_unlock(pipe);
1480 0 : return ret;
1481 : }
1482 :
1483 : /*
1484 : * Make sure there's writeable room. Wait for room if we can, otherwise
1485 : * return an appropriate error.
1486 : */
1487 0 : static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1488 : {
1489 : int ret;
1490 :
1491 : /*
1492 : * Check pipe occupancy without the inode lock first. This function
1493 : * is speculative anyways, so missing one is ok.
1494 : */
1495 0 : if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1496 : return 0;
1497 :
1498 0 : ret = 0;
1499 0 : pipe_lock(pipe);
1500 :
1501 0 : while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
1502 0 : if (!pipe->readers) {
1503 0 : send_sig(SIGPIPE, current, 0);
1504 0 : ret = -EPIPE;
1505 0 : break;
1506 : }
1507 0 : if (flags & SPLICE_F_NONBLOCK) {
1508 : ret = -EAGAIN;
1509 : break;
1510 : }
1511 0 : if (signal_pending(current)) {
1512 : ret = -ERESTARTSYS;
1513 : break;
1514 : }
1515 0 : pipe_wait_writable(pipe);
1516 : }
1517 :
1518 0 : pipe_unlock(pipe);
1519 0 : return ret;
1520 : }
1521 :
1522 : /*
1523 : * Splice contents of ipipe to opipe.
1524 : */
1525 0 : static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1526 : struct pipe_inode_info *opipe,
1527 : size_t len, unsigned int flags)
1528 : {
1529 : struct pipe_buffer *ibuf, *obuf;
1530 : unsigned int i_head, o_head;
1531 : unsigned int i_tail, o_tail;
1532 : unsigned int i_mask, o_mask;
1533 0 : int ret = 0;
1534 0 : bool input_wakeup = false;
1535 :
1536 :
1537 : retry:
1538 0 : ret = ipipe_prep(ipipe, flags);
1539 0 : if (ret)
1540 : return ret;
1541 :
1542 0 : ret = opipe_prep(opipe, flags);
1543 0 : if (ret)
1544 : return ret;
1545 :
1546 : /*
1547 : * Potential ABBA deadlock, work around it by ordering lock
1548 : * grabbing by pipe info address. Otherwise two different processes
1549 : * could deadlock (one doing tee from A -> B, the other from B -> A).
1550 : */
1551 0 : pipe_double_lock(ipipe, opipe);
1552 :
1553 0 : i_tail = ipipe->tail;
1554 0 : i_mask = ipipe->ring_size - 1;
1555 0 : o_head = opipe->head;
1556 0 : o_mask = opipe->ring_size - 1;
1557 :
1558 : do {
1559 : size_t o_len;
1560 :
1561 0 : if (!opipe->readers) {
1562 0 : send_sig(SIGPIPE, current, 0);
1563 0 : if (!ret)
1564 0 : ret = -EPIPE;
1565 : break;
1566 : }
1567 :
1568 0 : i_head = ipipe->head;
1569 0 : o_tail = opipe->tail;
1570 :
1571 0 : if (pipe_empty(i_head, i_tail) && !ipipe->writers)
1572 : break;
1573 :
1574 : /*
1575 : * Cannot make any progress, because either the input
1576 : * pipe is empty or the output pipe is full.
1577 : */
1578 0 : if (pipe_empty(i_head, i_tail) ||
1579 0 : pipe_full(o_head, o_tail, opipe->max_usage)) {
1580 : /* Already processed some buffers, break */
1581 0 : if (ret)
1582 : break;
1583 :
1584 0 : if (flags & SPLICE_F_NONBLOCK) {
1585 : ret = -EAGAIN;
1586 : break;
1587 : }
1588 :
1589 : /*
1590 : * We raced with another reader/writer and haven't
1591 : * managed to process any buffers. A zero return
1592 : * value means EOF, so retry instead.
1593 : */
1594 0 : pipe_unlock(ipipe);
1595 0 : pipe_unlock(opipe);
1596 0 : goto retry;
1597 : }
1598 :
1599 0 : ibuf = &ipipe->bufs[i_tail & i_mask];
1600 0 : obuf = &opipe->bufs[o_head & o_mask];
1601 :
1602 0 : if (len >= ibuf->len) {
1603 : /*
1604 : * Simply move the whole buffer from ipipe to opipe
1605 : */
1606 0 : *obuf = *ibuf;
1607 0 : ibuf->ops = NULL;
1608 0 : i_tail++;
1609 0 : ipipe->tail = i_tail;
1610 0 : input_wakeup = true;
1611 0 : o_len = obuf->len;
1612 0 : o_head++;
1613 0 : opipe->head = o_head;
1614 : } else {
1615 : /*
1616 : * Get a reference to this pipe buffer,
1617 : * so we can copy the contents over.
1618 : */
1619 0 : if (!pipe_buf_get(ipipe, ibuf)) {
1620 0 : if (ret == 0)
1621 0 : ret = -EFAULT;
1622 : break;
1623 : }
1624 0 : *obuf = *ibuf;
1625 :
1626 : /*
1627 : * Don't inherit the gift and merge flags, we need to
1628 : * prevent multiple steals of this page.
1629 : */
1630 0 : obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1631 0 : obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1632 :
1633 0 : obuf->len = len;
1634 0 : ibuf->offset += len;
1635 0 : ibuf->len -= len;
1636 0 : o_len = len;
1637 0 : o_head++;
1638 0 : opipe->head = o_head;
1639 : }
1640 0 : ret += o_len;
1641 0 : len -= o_len;
1642 0 : } while (len);
1643 :
1644 0 : pipe_unlock(ipipe);
1645 0 : pipe_unlock(opipe);
1646 :
1647 : /*
1648 : * If we put data in the output pipe, wakeup any potential readers.
1649 : */
1650 0 : if (ret > 0)
1651 0 : wakeup_pipe_readers(opipe);
1652 :
1653 0 : if (input_wakeup)
1654 0 : wakeup_pipe_writers(ipipe);
1655 :
1656 : return ret;
1657 : }
1658 :
1659 : /*
1660 : * Link contents of ipipe to opipe.
1661 : */
1662 0 : static int link_pipe(struct pipe_inode_info *ipipe,
1663 : struct pipe_inode_info *opipe,
1664 : size_t len, unsigned int flags)
1665 : {
1666 : struct pipe_buffer *ibuf, *obuf;
1667 : unsigned int i_head, o_head;
1668 : unsigned int i_tail, o_tail;
1669 : unsigned int i_mask, o_mask;
1670 0 : int ret = 0;
1671 :
1672 : /*
1673 : * Potential ABBA deadlock, work around it by ordering lock
1674 : * grabbing by pipe info address. Otherwise two different processes
1675 : * could deadlock (one doing tee from A -> B, the other from B -> A).
1676 : */
1677 0 : pipe_double_lock(ipipe, opipe);
1678 :
1679 0 : i_tail = ipipe->tail;
1680 0 : i_mask = ipipe->ring_size - 1;
1681 0 : o_head = opipe->head;
1682 0 : o_mask = opipe->ring_size - 1;
1683 :
1684 : do {
1685 0 : if (!opipe->readers) {
1686 0 : send_sig(SIGPIPE, current, 0);
1687 0 : if (!ret)
1688 0 : ret = -EPIPE;
1689 : break;
1690 : }
1691 :
1692 0 : i_head = ipipe->head;
1693 0 : o_tail = opipe->tail;
1694 :
1695 : /*
1696 : * If we have iterated all input buffers or run out of
1697 : * output room, break.
1698 : */
1699 0 : if (pipe_empty(i_head, i_tail) ||
1700 0 : pipe_full(o_head, o_tail, opipe->max_usage))
1701 : break;
1702 :
1703 0 : ibuf = &ipipe->bufs[i_tail & i_mask];
1704 0 : obuf = &opipe->bufs[o_head & o_mask];
1705 :
1706 : /*
1707 : * Get a reference to this pipe buffer,
1708 : * so we can copy the contents over.
1709 : */
1710 0 : if (!pipe_buf_get(ipipe, ibuf)) {
1711 0 : if (ret == 0)
1712 0 : ret = -EFAULT;
1713 : break;
1714 : }
1715 :
1716 0 : *obuf = *ibuf;
1717 :
1718 : /*
1719 : * Don't inherit the gift and merge flag, we need to prevent
1720 : * multiple steals of this page.
1721 : */
1722 0 : obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1723 0 : obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1724 :
1725 0 : if (obuf->len > len)
1726 0 : obuf->len = len;
1727 0 : ret += obuf->len;
1728 0 : len -= obuf->len;
1729 :
1730 0 : o_head++;
1731 0 : opipe->head = o_head;
1732 0 : i_tail++;
1733 0 : } while (len);
1734 :
1735 0 : pipe_unlock(ipipe);
1736 0 : pipe_unlock(opipe);
1737 :
1738 : /*
1739 : * If we put data in the output pipe, wakeup any potential readers.
1740 : */
1741 0 : if (ret > 0)
1742 0 : wakeup_pipe_readers(opipe);
1743 :
1744 0 : return ret;
1745 : }
1746 :
1747 : /*
1748 : * This is a tee(1) implementation that works on pipes. It doesn't copy
1749 : * any data, it simply references the 'in' pages on the 'out' pipe.
1750 : * The 'flags' used are the SPLICE_F_* variants, currently the only
1751 : * applicable one is SPLICE_F_NONBLOCK.
1752 : */
1753 0 : long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
1754 : {
1755 0 : struct pipe_inode_info *ipipe = get_pipe_info(in, true);
1756 0 : struct pipe_inode_info *opipe = get_pipe_info(out, true);
1757 0 : int ret = -EINVAL;
1758 :
1759 0 : if (unlikely(!(in->f_mode & FMODE_READ) ||
1760 : !(out->f_mode & FMODE_WRITE)))
1761 : return -EBADF;
1762 :
1763 : /*
1764 : * Duplicate the contents of ipipe to opipe without actually
1765 : * copying the data.
1766 : */
1767 0 : if (ipipe && opipe && ipipe != opipe) {
1768 0 : if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1769 0 : flags |= SPLICE_F_NONBLOCK;
1770 :
1771 : /*
1772 : * Keep going, unless we encounter an error. The ipipe/opipe
1773 : * ordering doesn't really matter.
1774 : */
1775 0 : ret = ipipe_prep(ipipe, flags);
1776 0 : if (!ret) {
1777 0 : ret = opipe_prep(opipe, flags);
1778 0 : if (!ret)
1779 0 : ret = link_pipe(ipipe, opipe, len, flags);
1780 : }
1781 : }
1782 :
1783 0 : return ret;
1784 : }
1785 :
1786 0 : SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1787 : {
1788 : struct fd in, out;
1789 : int error;
1790 :
1791 0 : if (unlikely(flags & ~SPLICE_F_ALL))
1792 : return -EINVAL;
1793 :
1794 0 : if (unlikely(!len))
1795 : return 0;
1796 :
1797 0 : error = -EBADF;
1798 0 : in = fdget(fdin);
1799 0 : if (in.file) {
1800 0 : out = fdget(fdout);
1801 0 : if (out.file) {
1802 0 : error = do_tee(in.file, out.file, len, flags);
1803 0 : fdput(out);
1804 : }
1805 0 : fdput(in);
1806 : }
1807 :
1808 0 : return error;
1809 : }
|
