Root/fs/splice.c

1/*
2 * "splice": joining two ropes together by interweaving their strands.
3 *
4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other.
7 *
8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer.
10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by
12 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs.
14 *
15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
18 *
19 */
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/splice.h>
24#include <linux/memcontrol.h>
25#include <linux/mm_inline.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
28#include <linux/buffer_head.h>
29#include <linux/module.h>
30#include <linux/syscalls.h>
31#include <linux/uio.h>
32#include <linux/security.h>
33#include <linux/gfp.h>
34
35/*
36 * Attempt to steal a page from a pipe buffer. This should perhaps go into
37 * a vm helper function, it's already simplified quite a bit by the
38 * addition of remove_mapping(). If success is returned, the caller may
39 * attempt to reuse this page for another destination.
40 */
41static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
42                     struct pipe_buffer *buf)
43{
44    struct page *page = buf->page;
45    struct address_space *mapping;
46
47    lock_page(page);
48
49    mapping = page_mapping(page);
50    if (mapping) {
51        WARN_ON(!PageUptodate(page));
52
53        /*
54         * At least for ext2 with nobh option, we need to wait on
55         * writeback completing on this page, since we'll remove it
56         * from the pagecache. Otherwise truncate wont wait on the
57         * page, allowing the disk blocks to be reused by someone else
58         * before we actually wrote our data to them. fs corruption
59         * ensues.
60         */
61        wait_on_page_writeback(page);
62
63        if (page_has_private(page) &&
64            !try_to_release_page(page, GFP_KERNEL))
65            goto out_unlock;
66
67        /*
68         * If we succeeded in removing the mapping, set LRU flag
69         * and return good.
70         */
71        if (remove_mapping(mapping, page)) {
72            buf->flags |= PIPE_BUF_FLAG_LRU;
73            return 0;
74        }
75    }
76
77    /*
78     * Raced with truncate or failed to remove page from current
79     * address space, unlock and return failure.
80     */
81out_unlock:
82    unlock_page(page);
83    return 1;
84}
85
86static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
87                    struct pipe_buffer *buf)
88{
89    page_cache_release(buf->page);
90    buf->flags &= ~PIPE_BUF_FLAG_LRU;
91}
92
93/*
94 * Check whether the contents of buf is OK to access. Since the content
95 * is a page cache page, IO may be in flight.
96 */
97static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
98                       struct pipe_buffer *buf)
99{
100    struct page *page = buf->page;
101    int err;
102
103    if (!PageUptodate(page)) {
104        lock_page(page);
105
106        /*
107         * Page got truncated/unhashed. This will cause a 0-byte
108         * splice, if this is the first page.
109         */
110        if (!page->mapping) {
111            err = -ENODATA;
112            goto error;
113        }
114
115        /*
116         * Uh oh, read-error from disk.
117         */
118        if (!PageUptodate(page)) {
119            err = -EIO;
120            goto error;
121        }
122
123        /*
124         * Page is ok afterall, we are done.
125         */
126        unlock_page(page);
127    }
128
129    return 0;
130error:
131    unlock_page(page);
132    return err;
133}
134
135static const struct pipe_buf_operations page_cache_pipe_buf_ops = {
136    .can_merge = 0,
137    .map = generic_pipe_buf_map,
138    .unmap = generic_pipe_buf_unmap,
139    .confirm = page_cache_pipe_buf_confirm,
140    .release = page_cache_pipe_buf_release,
141    .steal = page_cache_pipe_buf_steal,
142    .get = generic_pipe_buf_get,
143};
144
145static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
146                    struct pipe_buffer *buf)
147{
148    if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
149        return 1;
150
151    buf->flags |= PIPE_BUF_FLAG_LRU;
152    return generic_pipe_buf_steal(pipe, buf);
153}
154
155static const struct pipe_buf_operations user_page_pipe_buf_ops = {
156    .can_merge = 0,
157    .map = generic_pipe_buf_map,
158    .unmap = generic_pipe_buf_unmap,
159    .confirm = generic_pipe_buf_confirm,
160    .release = page_cache_pipe_buf_release,
161    .steal = user_page_pipe_buf_steal,
162    .get = generic_pipe_buf_get,
163};
164
165/**
166 * splice_to_pipe - fill passed data into a pipe
167 * @pipe: pipe to fill
168 * @spd: data to fill
169 *
170 * Description:
171 * @spd contains a map of pages and len/offset tuples, along with
172 * the struct pipe_buf_operations associated with these pages. This
173 * function will link that data to the pipe.
174 *
175 */
176ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
177               struct splice_pipe_desc *spd)
178{
179    unsigned int spd_pages = spd->nr_pages;
180    int ret, do_wakeup, page_nr;
181
182    ret = 0;
183    do_wakeup = 0;
184    page_nr = 0;
185
186    pipe_lock(pipe);
187
188    for (;;) {
189        if (!pipe->readers) {
190            send_sig(SIGPIPE, current, 0);
191            if (!ret)
192                ret = -EPIPE;
193            break;
194        }
195
196        if (pipe->nrbufs < PIPE_BUFFERS) {
197            int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
198            struct pipe_buffer *buf = pipe->bufs + newbuf;
199
200            buf->page = spd->pages[page_nr];
201            buf->offset = spd->partial[page_nr].offset;
202            buf->len = spd->partial[page_nr].len;
203            buf->private = spd->partial[page_nr].private;
204            buf->ops = spd->ops;
205            if (spd->flags & SPLICE_F_GIFT)
206                buf->flags |= PIPE_BUF_FLAG_GIFT;
207
208            pipe->nrbufs++;
209            page_nr++;
210            ret += buf->len;
211
212            if (pipe->inode)
213                do_wakeup = 1;
214
215            if (!--spd->nr_pages)
216                break;
217            if (pipe->nrbufs < PIPE_BUFFERS)
218                continue;
219
220            break;
221        }
222
223        if (spd->flags & SPLICE_F_NONBLOCK) {
224            if (!ret)
225                ret = -EAGAIN;
226            break;
227        }
228
229        if (signal_pending(current)) {
230            if (!ret)
231                ret = -ERESTARTSYS;
232            break;
233        }
234
235        if (do_wakeup) {
236            smp_mb();
237            if (waitqueue_active(&pipe->wait))
238                wake_up_interruptible_sync(&pipe->wait);
239            kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
240            do_wakeup = 0;
241        }
242
243        pipe->waiting_writers++;
244        pipe_wait(pipe);
245        pipe->waiting_writers--;
246    }
247
248    pipe_unlock(pipe);
249
250    if (do_wakeup) {
251        smp_mb();
252        if (waitqueue_active(&pipe->wait))
253            wake_up_interruptible(&pipe->wait);
254        kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
255    }
256
257    while (page_nr < spd_pages)
258        spd->spd_release(spd, page_nr++);
259
260    return ret;
261}
262
263static void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
264{
265    page_cache_release(spd->pages[i]);
266}
267
268static int
269__generic_file_splice_read(struct file *in, loff_t *ppos,
270               struct pipe_inode_info *pipe, size_t len,
271               unsigned int flags)
272{
273    struct address_space *mapping = in->f_mapping;
274    unsigned int loff, nr_pages, req_pages;
275    struct page *pages[PIPE_BUFFERS];
276    struct partial_page partial[PIPE_BUFFERS];
277    struct page *page;
278    pgoff_t index, end_index;
279    loff_t isize;
280    int error, page_nr;
281    struct splice_pipe_desc spd = {
282        .pages = pages,
283        .partial = partial,
284        .flags = flags,
285        .ops = &page_cache_pipe_buf_ops,
286        .spd_release = spd_release_page,
287    };
288
289    index = *ppos >> PAGE_CACHE_SHIFT;
290    loff = *ppos & ~PAGE_CACHE_MASK;
291    req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
292    nr_pages = min(req_pages, (unsigned)PIPE_BUFFERS);
293
294    /*
295     * Lookup the (hopefully) full range of pages we need.
296     */
297    spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
298    index += spd.nr_pages;
299
300    /*
301     * If find_get_pages_contig() returned fewer pages than we needed,
302     * readahead/allocate the rest and fill in the holes.
303     */
304    if (spd.nr_pages < nr_pages)
305        page_cache_sync_readahead(mapping, &in->f_ra, in,
306                index, req_pages - spd.nr_pages);
307
308    error = 0;
309    while (spd.nr_pages < nr_pages) {
310        /*
311         * Page could be there, find_get_pages_contig() breaks on
312         * the first hole.
313         */
314        page = find_get_page(mapping, index);
315        if (!page) {
316            /*
317             * page didn't exist, allocate one.
318             */
319            page = page_cache_alloc_cold(mapping);
320            if (!page)
321                break;
322
323            error = add_to_page_cache_lru(page, mapping, index,
324                        mapping_gfp_mask(mapping));
325            if (unlikely(error)) {
326                page_cache_release(page);
327                if (error == -EEXIST)
328                    continue;
329                break;
330            }
331            /*
332             * add_to_page_cache() locks the page, unlock it
333             * to avoid convoluting the logic below even more.
334             */
335            unlock_page(page);
336        }
337
338        pages[spd.nr_pages++] = page;
339        index++;
340    }
341
342    /*
343     * Now loop over the map and see if we need to start IO on any
344     * pages, fill in the partial map, etc.
345     */
346    index = *ppos >> PAGE_CACHE_SHIFT;
347    nr_pages = spd.nr_pages;
348    spd.nr_pages = 0;
349    for (page_nr = 0; page_nr < nr_pages; page_nr++) {
350        unsigned int this_len;
351
352        if (!len)
353            break;
354
355        /*
356         * this_len is the max we'll use from this page
357         */
358        this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
359        page = pages[page_nr];
360
361        if (PageReadahead(page))
362            page_cache_async_readahead(mapping, &in->f_ra, in,
363                    page, index, req_pages - page_nr);
364
365        /*
366         * If the page isn't uptodate, we may need to start io on it
367         */
368        if (!PageUptodate(page)) {
369            /*
370             * If in nonblock mode then dont block on waiting
371             * for an in-flight io page
372             */
373            if (flags & SPLICE_F_NONBLOCK) {
374                if (!trylock_page(page)) {
375                    error = -EAGAIN;
376                    break;
377                }
378            } else
379                lock_page(page);
380
381            /*
382             * Page was truncated, or invalidated by the
383             * filesystem. Redo the find/create, but this time the
384             * page is kept locked, so there's no chance of another
385             * race with truncate/invalidate.
386             */
387            if (!page->mapping) {
388                unlock_page(page);
389                page = find_or_create_page(mapping, index,
390                        mapping_gfp_mask(mapping));
391
392                if (!page) {
393                    error = -ENOMEM;
394                    break;
395                }
396                page_cache_release(pages[page_nr]);
397                pages[page_nr] = page;
398            }
399            /*
400             * page was already under io and is now done, great
401             */
402            if (PageUptodate(page)) {
403                unlock_page(page);
404                goto fill_it;
405            }
406
407            /*
408             * need to read in the page
409             */
410            error = mapping->a_ops->readpage(in, page);
411            if (unlikely(error)) {
412                /*
413                 * We really should re-lookup the page here,
414                 * but it complicates things a lot. Instead
415                 * lets just do what we already stored, and
416                 * we'll get it the next time we are called.
417                 */
418                if (error == AOP_TRUNCATED_PAGE)
419                    error = 0;
420
421                break;
422            }
423        }
424fill_it:
425        /*
426         * i_size must be checked after PageUptodate.
427         */
428        isize = i_size_read(mapping->host);
429        end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
430        if (unlikely(!isize || index > end_index))
431            break;
432
433        /*
434         * if this is the last page, see if we need to shrink
435         * the length and stop
436         */
437        if (end_index == index) {
438            unsigned int plen;
439
440            /*
441             * max good bytes in this page
442             */
443            plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
444            if (plen <= loff)
445                break;
446
447            /*
448             * force quit after adding this page
449             */
450            this_len = min(this_len, plen - loff);
451            len = this_len;
452        }
453
454        partial[page_nr].offset = loff;
455        partial[page_nr].len = this_len;
456        len -= this_len;
457        loff = 0;
458        spd.nr_pages++;
459        index++;
460    }
461
462    /*
463     * Release any pages at the end, if we quit early. 'page_nr' is how far
464     * we got, 'nr_pages' is how many pages are in the map.
465     */
466    while (page_nr < nr_pages)
467        page_cache_release(pages[page_nr++]);
468    in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT;
469
470    if (spd.nr_pages)
471        return splice_to_pipe(pipe, &spd);
472
473    return error;
474}
475
476/**
477 * generic_file_splice_read - splice data from file to a pipe
478 * @in: file to splice from
479 * @ppos: position in @in
480 * @pipe: pipe to splice to
481 * @len: number of bytes to splice
482 * @flags: splice modifier flags
483 *
484 * Description:
485 * Will read pages from given file and fill them into a pipe. Can be
486 * used as long as the address_space operations for the source implements
487 * a readpage() hook.
488 *
489 */
490ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
491                 struct pipe_inode_info *pipe, size_t len,
492                 unsigned int flags)
493{
494    loff_t isize, left;
495    int ret;
496
497    isize = i_size_read(in->f_mapping->host);
498    if (unlikely(*ppos >= isize))
499        return 0;
500
501    left = isize - *ppos;
502    if (unlikely(left < len))
503        len = left;
504
505    ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
506    if (ret > 0) {
507        *ppos += ret;
508        file_accessed(in);
509    }
510
511    return ret;
512}
513EXPORT_SYMBOL(generic_file_splice_read);
514
515static const struct pipe_buf_operations default_pipe_buf_ops = {
516    .can_merge = 0,
517    .map = generic_pipe_buf_map,
518    .unmap = generic_pipe_buf_unmap,
519    .confirm = generic_pipe_buf_confirm,
520    .release = generic_pipe_buf_release,
521    .steal = generic_pipe_buf_steal,
522    .get = generic_pipe_buf_get,
523};
524
525static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
526                unsigned long vlen, loff_t offset)
527{
528    mm_segment_t old_fs;
529    loff_t pos = offset;
530    ssize_t res;
531
532    old_fs = get_fs();
533    set_fs(get_ds());
534    /* The cast to a user pointer is valid due to the set_fs() */
535    res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
536    set_fs(old_fs);
537
538    return res;
539}
540
541static ssize_t kernel_write(struct file *file, const char *buf, size_t count,
542                loff_t pos)
543{
544    mm_segment_t old_fs;
545    ssize_t res;
546
547    old_fs = get_fs();
548    set_fs(get_ds());
549    /* The cast to a user pointer is valid due to the set_fs() */
550    res = vfs_write(file, (const char __user *)buf, count, &pos);
551    set_fs(old_fs);
552
553    return res;
554}
555
556ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
557                 struct pipe_inode_info *pipe, size_t len,
558                 unsigned int flags)
559{
560    unsigned int nr_pages;
561    unsigned int nr_freed;
562    size_t offset;
563    struct page *pages[PIPE_BUFFERS];
564    struct partial_page partial[PIPE_BUFFERS];
565    struct iovec vec[PIPE_BUFFERS];
566    pgoff_t index;
567    ssize_t res;
568    size_t this_len;
569    int error;
570    int i;
571    struct splice_pipe_desc spd = {
572        .pages = pages,
573        .partial = partial,
574        .flags = flags,
575        .ops = &default_pipe_buf_ops,
576        .spd_release = spd_release_page,
577    };
578
579    index = *ppos >> PAGE_CACHE_SHIFT;
580    offset = *ppos & ~PAGE_CACHE_MASK;
581    nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
582
583    for (i = 0; i < nr_pages && i < PIPE_BUFFERS && len; i++) {
584        struct page *page;
585
586        page = alloc_page(GFP_USER);
587        error = -ENOMEM;
588        if (!page)
589            goto err;
590
591        this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
592        vec[i].iov_base = (void __user *) page_address(page);
593        vec[i].iov_len = this_len;
594        pages[i] = page;
595        spd.nr_pages++;
596        len -= this_len;
597        offset = 0;
598    }
599
600    res = kernel_readv(in, vec, spd.nr_pages, *ppos);
601    if (res < 0) {
602        error = res;
603        goto err;
604    }
605
606    error = 0;
607    if (!res)
608        goto err;
609
610    nr_freed = 0;
611    for (i = 0; i < spd.nr_pages; i++) {
612        this_len = min_t(size_t, vec[i].iov_len, res);
613        partial[i].offset = 0;
614        partial[i].len = this_len;
615        if (!this_len) {
616            __free_page(pages[i]);
617            pages[i] = NULL;
618            nr_freed++;
619        }
620        res -= this_len;
621    }
622    spd.nr_pages -= nr_freed;
623
624    res = splice_to_pipe(pipe, &spd);
625    if (res > 0)
626        *ppos += res;
627
628    return res;
629
630err:
631    for (i = 0; i < spd.nr_pages; i++)
632        __free_page(pages[i]);
633
634    return error;
635}
636EXPORT_SYMBOL(default_file_splice_read);
637
638/*
639 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
640 * using sendpage(). Return the number of bytes sent.
641 */
642static int pipe_to_sendpage(struct pipe_inode_info *pipe,
643                struct pipe_buffer *buf, struct splice_desc *sd)
644{
645    struct file *file = sd->u.file;
646    loff_t pos = sd->pos;
647    int ret, more;
648
649    ret = buf->ops->confirm(pipe, buf);
650    if (!ret) {
651        more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
652        if (file->f_op && file->f_op->sendpage)
653            ret = file->f_op->sendpage(file, buf->page, buf->offset,
654                           sd->len, &pos, more);
655        else
656            ret = -EINVAL;
657    }
658
659    return ret;
660}
661
662/*
663 * This is a little more tricky than the file -> pipe splicing. There are
664 * basically three cases:
665 *
666 * - Destination page already exists in the address space and there
667 * are users of it. For that case we have no other option that
668 * copying the data. Tough luck.
669 * - Destination page already exists in the address space, but there
670 * are no users of it. Make sure it's uptodate, then drop it. Fall
671 * through to last case.
672 * - Destination page does not exist, we can add the pipe page to
673 * the page cache and avoid the copy.
674 *
675 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
676 * sd->flags), we attempt to migrate pages from the pipe to the output
677 * file address space page cache. This is possible if no one else has
678 * the pipe page referenced outside of the pipe and page cache. If
679 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
680 * a new page in the output file page cache and fill/dirty that.
681 */
682int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
683         struct splice_desc *sd)
684{
685    struct file *file = sd->u.file;
686    struct address_space *mapping = file->f_mapping;
687    unsigned int offset, this_len;
688    struct page *page;
689    void *fsdata;
690    int ret;
691
692    /*
693     * make sure the data in this buffer is uptodate
694     */
695    ret = buf->ops->confirm(pipe, buf);
696    if (unlikely(ret))
697        return ret;
698
699    offset = sd->pos & ~PAGE_CACHE_MASK;
700
701    this_len = sd->len;
702    if (this_len + offset > PAGE_CACHE_SIZE)
703        this_len = PAGE_CACHE_SIZE - offset;
704
705    ret = pagecache_write_begin(file, mapping, sd->pos, this_len,
706                AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata);
707    if (unlikely(ret))
708        goto out;
709
710    if (buf->page != page) {
711        /*
712         * Careful, ->map() uses KM_USER0!
713         */
714        char *src = buf->ops->map(pipe, buf, 1);
715        char *dst = kmap_atomic(page, KM_USER1);
716
717        memcpy(dst + offset, src + buf->offset, this_len);
718        flush_dcache_page(page);
719        kunmap_atomic(dst, KM_USER1);
720        buf->ops->unmap(pipe, buf, src);
721    }
722    ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len,
723                page, fsdata);
724out:
725    return ret;
726}
727EXPORT_SYMBOL(pipe_to_file);
728
729static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
730{
731    smp_mb();
732    if (waitqueue_active(&pipe->wait))
733        wake_up_interruptible(&pipe->wait);
734    kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
735}
736
737/**
738 * splice_from_pipe_feed - feed available data from a pipe to a file
739 * @pipe: pipe to splice from
740 * @sd: information to @actor
741 * @actor: handler that splices the data
742 *
743 * Description:
744 * This function loops over the pipe and calls @actor to do the
745 * actual moving of a single struct pipe_buffer to the desired
746 * destination. It returns when there's no more buffers left in
747 * the pipe or if the requested number of bytes (@sd->total_len)
748 * have been copied. It returns a positive number (one) if the
749 * pipe needs to be filled with more data, zero if the required
750 * number of bytes have been copied and -errno on error.
751 *
752 * This, together with splice_from_pipe_{begin,end,next}, may be
753 * used to implement the functionality of __splice_from_pipe() when
754 * locking is required around copying the pipe buffers to the
755 * destination.
756 */
757int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
758              splice_actor *actor)
759{
760    int ret;
761
762    while (pipe->nrbufs) {
763        struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
764        const struct pipe_buf_operations *ops = buf->ops;
765
766        sd->len = buf->len;
767        if (sd->len > sd->total_len)
768            sd->len = sd->total_len;
769
770        ret = actor(pipe, buf, sd);
771        if (ret <= 0) {
772            if (ret == -ENODATA)
773                ret = 0;
774            return ret;
775        }
776        buf->offset += ret;
777        buf->len -= ret;
778
779        sd->num_spliced += ret;
780        sd->len -= ret;
781        sd->pos += ret;
782        sd->total_len -= ret;
783
784        if (!buf->len) {
785            buf->ops = NULL;
786            ops->release(pipe, buf);
787            pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
788            pipe->nrbufs--;
789            if (pipe->inode)
790                sd->need_wakeup = true;
791        }
792
793        if (!sd->total_len)
794            return 0;
795    }
796
797    return 1;
798}
799EXPORT_SYMBOL(splice_from_pipe_feed);
800
801/**
802 * splice_from_pipe_next - wait for some data to splice from
803 * @pipe: pipe to splice from
804 * @sd: information about the splice operation
805 *
806 * Description:
807 * This function will wait for some data and return a positive
808 * value (one) if pipe buffers are available. It will return zero
809 * or -errno if no more data needs to be spliced.
810 */
811int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
812{
813    while (!pipe->nrbufs) {
814        if (!pipe->writers)
815            return 0;
816
817        if (!pipe->waiting_writers && sd->num_spliced)
818            return 0;
819
820        if (sd->flags & SPLICE_F_NONBLOCK)
821            return -EAGAIN;
822
823        if (signal_pending(current))
824            return -ERESTARTSYS;
825
826        if (sd->need_wakeup) {
827            wakeup_pipe_writers(pipe);
828            sd->need_wakeup = false;
829        }
830
831        pipe_wait(pipe);
832    }
833
834    return 1;
835}
836EXPORT_SYMBOL(splice_from_pipe_next);
837
838/**
839 * splice_from_pipe_begin - start splicing from pipe
840 * @sd: information about the splice operation
841 *
842 * Description:
843 * This function should be called before a loop containing
844 * splice_from_pipe_next() and splice_from_pipe_feed() to
845 * initialize the necessary fields of @sd.
846 */
847void splice_from_pipe_begin(struct splice_desc *sd)
848{
849    sd->num_spliced = 0;
850    sd->need_wakeup = false;
851}
852EXPORT_SYMBOL(splice_from_pipe_begin);
853
854/**
855 * splice_from_pipe_end - finish splicing from pipe
856 * @pipe: pipe to splice from
857 * @sd: information about the splice operation
858 *
859 * Description:
860 * This function will wake up pipe writers if necessary. It should
861 * be called after a loop containing splice_from_pipe_next() and
862 * splice_from_pipe_feed().
863 */
864void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
865{
866    if (sd->need_wakeup)
867        wakeup_pipe_writers(pipe);
868}
869EXPORT_SYMBOL(splice_from_pipe_end);
870
871/**
872 * __splice_from_pipe - splice data from a pipe to given actor
873 * @pipe: pipe to splice from
874 * @sd: information to @actor
875 * @actor: handler that splices the data
876 *
877 * Description:
878 * This function does little more than loop over the pipe and call
879 * @actor to do the actual moving of a single struct pipe_buffer to
880 * the desired destination. See pipe_to_file, pipe_to_sendpage, or
881 * pipe_to_user.
882 *
883 */
884ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
885               splice_actor *actor)
886{
887    int ret;
888
889    splice_from_pipe_begin(sd);
890    do {
891        ret = splice_from_pipe_next(pipe, sd);
892        if (ret > 0)
893            ret = splice_from_pipe_feed(pipe, sd, actor);
894    } while (ret > 0);
895    splice_from_pipe_end(pipe, sd);
896
897    return sd->num_spliced ? sd->num_spliced : ret;
898}
899EXPORT_SYMBOL(__splice_from_pipe);
900
901/**
902 * splice_from_pipe - splice data from a pipe to a file
903 * @pipe: pipe to splice from
904 * @out: file to splice to
905 * @ppos: position in @out
906 * @len: how many bytes to splice
907 * @flags: splice modifier flags
908 * @actor: handler that splices the data
909 *
910 * Description:
911 * See __splice_from_pipe. This function locks the pipe inode,
912 * otherwise it's identical to __splice_from_pipe().
913 *
914 */
915ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
916             loff_t *ppos, size_t len, unsigned int flags,
917             splice_actor *actor)
918{
919    ssize_t ret;
920    struct splice_desc sd = {
921        .total_len = len,
922        .flags = flags,
923        .pos = *ppos,
924        .u.file = out,
925    };
926
927    pipe_lock(pipe);
928    ret = __splice_from_pipe(pipe, &sd, actor);
929    pipe_unlock(pipe);
930
931    return ret;
932}
933
934/**
935 * generic_file_splice_write - splice data from a pipe to a file
936 * @pipe: pipe info
937 * @out: file to write to
938 * @ppos: position in @out
939 * @len: number of bytes to splice
940 * @flags: splice modifier flags
941 *
942 * Description:
943 * Will either move or copy pages (determined by @flags options) from
944 * the given pipe inode to the given file.
945 *
946 */
947ssize_t
948generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
949              loff_t *ppos, size_t len, unsigned int flags)
950{
951    struct address_space *mapping = out->f_mapping;
952    struct inode *inode = mapping->host;
953    struct splice_desc sd = {
954        .total_len = len,
955        .flags = flags,
956        .pos = *ppos,
957        .u.file = out,
958    };
959    ssize_t ret;
960
961    pipe_lock(pipe);
962
963    splice_from_pipe_begin(&sd);
964    do {
965        ret = splice_from_pipe_next(pipe, &sd);
966        if (ret <= 0)
967            break;
968
969        mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD);
970        ret = file_remove_suid(out);
971        if (!ret) {
972            file_update_time(out);
973            ret = splice_from_pipe_feed(pipe, &sd, pipe_to_file);
974        }
975        mutex_unlock(&inode->i_mutex);
976    } while (ret > 0);
977    splice_from_pipe_end(pipe, &sd);
978
979    pipe_unlock(pipe);
980
981    if (sd.num_spliced)
982        ret = sd.num_spliced;
983
984    if (ret > 0) {
985        unsigned long nr_pages;
986        int err;
987
988        nr_pages = (ret + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
989
990        err = generic_write_sync(out, *ppos, ret);
991        if (err)
992            ret = err;
993        else
994            *ppos += ret;
995        balance_dirty_pages_ratelimited_nr(mapping, nr_pages);
996    }
997
998    return ret;
999}
1000
1001EXPORT_SYMBOL(generic_file_splice_write);
1002
1003static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1004              struct splice_desc *sd)
1005{
1006    int ret;
1007    void *data;
1008
1009    ret = buf->ops->confirm(pipe, buf);
1010    if (ret)
1011        return ret;
1012
1013    data = buf->ops->map(pipe, buf, 0);
1014    ret = kernel_write(sd->u.file, data + buf->offset, sd->len, sd->pos);
1015    buf->ops->unmap(pipe, buf, data);
1016
1017    return ret;
1018}
1019
1020static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
1021                     struct file *out, loff_t *ppos,
1022                     size_t len, unsigned int flags)
1023{
1024    ssize_t ret;
1025
1026    ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
1027    if (ret > 0)
1028        *ppos += ret;
1029
1030    return ret;
1031}
1032
1033/**
1034 * generic_splice_sendpage - splice data from a pipe to a socket
1035 * @pipe: pipe to splice from
1036 * @out: socket to write to
1037 * @ppos: position in @out
1038 * @len: number of bytes to splice
1039 * @flags: splice modifier flags
1040 *
1041 * Description:
1042 * Will send @len bytes from the pipe to a network socket. No data copying
1043 * is involved.
1044 *
1045 */
1046ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
1047                loff_t *ppos, size_t len, unsigned int flags)
1048{
1049    return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
1050}
1051
1052EXPORT_SYMBOL(generic_splice_sendpage);
1053
1054/*
1055 * Attempt to initiate a splice from pipe to file.
1056 */
1057static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
1058               loff_t *ppos, size_t len, unsigned int flags)
1059{
1060    ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
1061                loff_t *, size_t, unsigned int);
1062    int ret;
1063
1064    if (unlikely(!(out->f_mode & FMODE_WRITE)))
1065        return -EBADF;
1066
1067    if (unlikely(out->f_flags & O_APPEND))
1068        return -EINVAL;
1069
1070    ret = rw_verify_area(WRITE, out, ppos, len);
1071    if (unlikely(ret < 0))
1072        return ret;
1073
1074    if (out->f_op && out->f_op->splice_write)
1075        splice_write = out->f_op->splice_write;
1076    else
1077        splice_write = default_file_splice_write;
1078
1079    return splice_write(pipe, out, ppos, len, flags);
1080}
1081
1082/*
1083 * Attempt to initiate a splice from a file to a pipe.
1084 */
1085static long do_splice_to(struct file *in, loff_t *ppos,
1086             struct pipe_inode_info *pipe, size_t len,
1087             unsigned int flags)
1088{
1089    ssize_t (*splice_read)(struct file *, loff_t *,
1090                   struct pipe_inode_info *, size_t, unsigned int);
1091    int ret;
1092
1093    if (unlikely(!(in->f_mode & FMODE_READ)))
1094        return -EBADF;
1095
1096    ret = rw_verify_area(READ, in, ppos, len);
1097    if (unlikely(ret < 0))
1098        return ret;
1099
1100    if (in->f_op && in->f_op->splice_read)
1101        splice_read = in->f_op->splice_read;
1102    else
1103        splice_read = default_file_splice_read;
1104
1105    return splice_read(in, ppos, pipe, len, flags);
1106}
1107
1108/**
1109 * splice_direct_to_actor - splices data directly between two non-pipes
1110 * @in: file to splice from
1111 * @sd: actor information on where to splice to
1112 * @actor: handles the data splicing
1113 *
1114 * Description:
1115 * This is a special case helper to splice directly between two
1116 * points, without requiring an explicit pipe. Internally an allocated
1117 * pipe is cached in the process, and reused during the lifetime of
1118 * that process.
1119 *
1120 */
1121ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1122                   splice_direct_actor *actor)
1123{
1124    struct pipe_inode_info *pipe;
1125    long ret, bytes;
1126    umode_t i_mode;
1127    size_t len;
1128    int i, flags;
1129
1130    /*
1131     * We require the input being a regular file, as we don't want to
1132     * randomly drop data for eg socket -> socket splicing. Use the
1133     * piped splicing for that!
1134     */
1135    i_mode = in->f_path.dentry->d_inode->i_mode;
1136    if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
1137        return -EINVAL;
1138
1139    /*
1140     * neither in nor out is a pipe, setup an internal pipe attached to
1141     * 'out' and transfer the wanted data from 'in' to 'out' through that
1142     */
1143    pipe = current->splice_pipe;
1144    if (unlikely(!pipe)) {
1145        pipe = alloc_pipe_info(NULL);
1146        if (!pipe)
1147            return -ENOMEM;
1148
1149        /*
1150         * We don't have an immediate reader, but we'll read the stuff
1151         * out of the pipe right after the splice_to_pipe(). So set
1152         * PIPE_READERS appropriately.
1153         */
1154        pipe->readers = 1;
1155
1156        current->splice_pipe = pipe;
1157    }
1158
1159    /*
1160     * Do the splice.
1161     */
1162    ret = 0;
1163    bytes = 0;
1164    len = sd->total_len;
1165    flags = sd->flags;
1166
1167    /*
1168     * Don't block on output, we have to drain the direct pipe.
1169     */
1170    sd->flags &= ~SPLICE_F_NONBLOCK;
1171
1172    while (len) {
1173        size_t read_len;
1174        loff_t pos = sd->pos, prev_pos = pos;
1175
1176        ret = do_splice_to(in, &pos, pipe, len, flags);
1177        if (unlikely(ret <= 0))
1178            goto out_release;
1179
1180        read_len = ret;
1181        sd->total_len = read_len;
1182
1183        /*
1184         * NOTE: nonblocking mode only applies to the input. We
1185         * must not do the output in nonblocking mode as then we
1186         * could get stuck data in the internal pipe:
1187         */
1188        ret = actor(pipe, sd);
1189        if (unlikely(ret <= 0)) {
1190            sd->pos = prev_pos;
1191            goto out_release;
1192        }
1193
1194        bytes += ret;
1195        len -= ret;
1196        sd->pos = pos;
1197
1198        if (ret < read_len) {
1199            sd->pos = prev_pos + ret;
1200            goto out_release;
1201        }
1202    }
1203
1204done:
1205    pipe->nrbufs = pipe->curbuf = 0;
1206    file_accessed(in);
1207    return bytes;
1208
1209out_release:
1210    /*
1211     * If we did an incomplete transfer we must release
1212     * the pipe buffers in question:
1213     */
1214    for (i = 0; i < PIPE_BUFFERS; i++) {
1215        struct pipe_buffer *buf = pipe->bufs + i;
1216
1217        if (buf->ops) {
1218            buf->ops->release(pipe, buf);
1219            buf->ops = NULL;
1220        }
1221    }
1222
1223    if (!bytes)
1224        bytes = ret;
1225
1226    goto done;
1227}
1228EXPORT_SYMBOL(splice_direct_to_actor);
1229
1230static int direct_splice_actor(struct pipe_inode_info *pipe,
1231                   struct splice_desc *sd)
1232{
1233    struct file *file = sd->u.file;
1234
1235    return do_splice_from(pipe, file, &sd->pos, sd->total_len, sd->flags);
1236}
1237
1238/**
1239 * do_splice_direct - splices data directly between two files
1240 * @in: file to splice from
1241 * @ppos: input file offset
1242 * @out: file to splice to
1243 * @len: number of bytes to splice
1244 * @flags: splice modifier flags
1245 *
1246 * Description:
1247 * For use by do_sendfile(). splice can easily emulate sendfile, but
1248 * doing it in the application would incur an extra system call
1249 * (splice in + splice out, as compared to just sendfile()). So this helper
1250 * can splice directly through a process-private pipe.
1251 *
1252 */
1253long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1254              size_t len, unsigned int flags)
1255{
1256    struct splice_desc sd = {
1257        .len = len,
1258        .total_len = len,
1259        .flags = flags,
1260        .pos = *ppos,
1261        .u.file = out,
1262    };
1263    long ret;
1264
1265    ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1266    if (ret > 0)
1267        *ppos = sd.pos;
1268
1269    return ret;
1270}
1271
1272static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1273                   struct pipe_inode_info *opipe,
1274                   size_t len, unsigned int flags);
1275/*
1276 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
1277 * location, so checking ->i_pipe is not enough to verify that this is a
1278 * pipe.
1279 */
1280static inline struct pipe_inode_info *pipe_info(struct inode *inode)
1281{
1282    if (S_ISFIFO(inode->i_mode))
1283        return inode->i_pipe;
1284
1285    return NULL;
1286}
1287
1288/*
1289 * Determine where to splice to/from.
1290 */
1291static long do_splice(struct file *in, loff_t __user *off_in,
1292              struct file *out, loff_t __user *off_out,
1293              size_t len, unsigned int flags)
1294{
1295    struct pipe_inode_info *ipipe;
1296    struct pipe_inode_info *opipe;
1297    loff_t offset, *off;
1298    long ret;
1299
1300    ipipe = pipe_info(in->f_path.dentry->d_inode);
1301    opipe = pipe_info(out->f_path.dentry->d_inode);
1302
1303    if (ipipe && opipe) {
1304        if (off_in || off_out)
1305            return -ESPIPE;
1306
1307        if (!(in->f_mode & FMODE_READ))
1308            return -EBADF;
1309
1310        if (!(out->f_mode & FMODE_WRITE))
1311            return -EBADF;
1312
1313        /* Splicing to self would be fun, but... */
1314        if (ipipe == opipe)
1315            return -EINVAL;
1316
1317        return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1318    }
1319
1320    if (ipipe) {
1321        if (off_in)
1322            return -ESPIPE;
1323        if (off_out) {
1324            if (!out->f_op || !out->f_op->llseek ||
1325                out->f_op->llseek == no_llseek)
1326                return -EINVAL;
1327            if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1328                return -EFAULT;
1329            off = &offset;
1330        } else
1331            off = &out->f_pos;
1332
1333        ret = do_splice_from(ipipe, out, off, len, flags);
1334
1335        if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1336            ret = -EFAULT;
1337
1338        return ret;
1339    }
1340
1341    if (opipe) {
1342        if (off_out)
1343            return -ESPIPE;
1344        if (off_in) {
1345            if (!in->f_op || !in->f_op->llseek ||
1346                in->f_op->llseek == no_llseek)
1347                return -EINVAL;
1348            if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1349                return -EFAULT;
1350            off = &offset;
1351        } else
1352            off = &in->f_pos;
1353
1354        ret = do_splice_to(in, off, opipe, len, flags);
1355
1356        if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1357            ret = -EFAULT;
1358
1359        return ret;
1360    }
1361
1362    return -EINVAL;
1363}
1364
1365/*
1366 * Map an iov into an array of pages and offset/length tupples. With the
1367 * partial_page structure, we can map several non-contiguous ranges into
1368 * our ones pages[] map instead of splitting that operation into pieces.
1369 * Could easily be exported as a generic helper for other users, in which
1370 * case one would probably want to add a 'max_nr_pages' parameter as well.
1371 */
1372static int get_iovec_page_array(const struct iovec __user *iov,
1373                unsigned int nr_vecs, struct page **pages,
1374                struct partial_page *partial, int aligned)
1375{
1376    int buffers = 0, error = 0;
1377
1378    while (nr_vecs) {
1379        unsigned long off, npages;
1380        struct iovec entry;
1381        void __user *base;
1382        size_t len;
1383        int i;
1384
1385        error = -EFAULT;
1386        if (copy_from_user(&entry, iov, sizeof(entry)))
1387            break;
1388
1389        base = entry.iov_base;
1390        len = entry.iov_len;
1391
1392        /*
1393         * Sanity check this iovec. 0 read succeeds.
1394         */
1395        error = 0;
1396        if (unlikely(!len))
1397            break;
1398        error = -EFAULT;
1399        if (!access_ok(VERIFY_READ, base, len))
1400            break;
1401
1402        /*
1403         * Get this base offset and number of pages, then map
1404         * in the user pages.
1405         */
1406        off = (unsigned long) base & ~PAGE_MASK;
1407
1408        /*
1409         * If asked for alignment, the offset must be zero and the
1410         * length a multiple of the PAGE_SIZE.
1411         */
1412        error = -EINVAL;
1413        if (aligned && (off || len & ~PAGE_MASK))
1414            break;
1415
1416        npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1417        if (npages > PIPE_BUFFERS - buffers)
1418            npages = PIPE_BUFFERS - buffers;
1419
1420        error = get_user_pages_fast((unsigned long)base, npages,
1421                    0, &pages[buffers]);
1422
1423        if (unlikely(error <= 0))
1424            break;
1425
1426        /*
1427         * Fill this contiguous range into the partial page map.
1428         */
1429        for (i = 0; i < error; i++) {
1430            const int plen = min_t(size_t, len, PAGE_SIZE - off);
1431
1432            partial[buffers].offset = off;
1433            partial[buffers].len = plen;
1434
1435            off = 0;
1436            len -= plen;
1437            buffers++;
1438        }
1439
1440        /*
1441         * We didn't complete this iov, stop here since it probably
1442         * means we have to move some of this into a pipe to
1443         * be able to continue.
1444         */
1445        if (len)
1446            break;
1447
1448        /*
1449         * Don't continue if we mapped fewer pages than we asked for,
1450         * or if we mapped the max number of pages that we have
1451         * room for.
1452         */
1453        if (error < npages || buffers == PIPE_BUFFERS)
1454            break;
1455
1456        nr_vecs--;
1457        iov++;
1458    }
1459
1460    if (buffers)
1461        return buffers;
1462
1463    return error;
1464}
1465
1466static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1467            struct splice_desc *sd)
1468{
1469    char *src;
1470    int ret;
1471
1472    ret = buf->ops->confirm(pipe, buf);
1473    if (unlikely(ret))
1474        return ret;
1475
1476    /*
1477     * See if we can use the atomic maps, by prefaulting in the
1478     * pages and doing an atomic copy
1479     */
1480    if (!fault_in_pages_writeable(sd->u.userptr, sd->len)) {
1481        src = buf->ops->map(pipe, buf, 1);
1482        ret = __copy_to_user_inatomic(sd->u.userptr, src + buf->offset,
1483                            sd->len);
1484        buf->ops->unmap(pipe, buf, src);
1485        if (!ret) {
1486            ret = sd->len;
1487            goto out;
1488        }
1489    }
1490
1491    /*
1492     * No dice, use slow non-atomic map and copy
1493      */
1494    src = buf->ops->map(pipe, buf, 0);
1495
1496    ret = sd->len;
1497    if (copy_to_user(sd->u.userptr, src + buf->offset, sd->len))
1498        ret = -EFAULT;
1499
1500    buf->ops->unmap(pipe, buf, src);
1501out:
1502    if (ret > 0)
1503        sd->u.userptr += ret;
1504    return ret;
1505}
1506
1507/*
1508 * For lack of a better implementation, implement vmsplice() to userspace
1509 * as a simple copy of the pipes pages to the user iov.
1510 */
1511static long vmsplice_to_user(struct file *file, const struct iovec __user *iov,
1512                 unsigned long nr_segs, unsigned int flags)
1513{
1514    struct pipe_inode_info *pipe;
1515    struct splice_desc sd;
1516    ssize_t size;
1517    int error;
1518    long ret;
1519
1520    pipe = pipe_info(file->f_path.dentry->d_inode);
1521    if (!pipe)
1522        return -EBADF;
1523
1524    pipe_lock(pipe);
1525
1526    error = ret = 0;
1527    while (nr_segs) {
1528        void __user *base;
1529        size_t len;
1530
1531        /*
1532         * Get user address base and length for this iovec.
1533         */
1534        error = get_user(base, &iov->iov_base);
1535        if (unlikely(error))
1536            break;
1537        error = get_user(len, &iov->iov_len);
1538        if (unlikely(error))
1539            break;
1540
1541        /*
1542         * Sanity check this iovec. 0 read succeeds.
1543         */
1544        if (unlikely(!len))
1545            break;
1546        if (unlikely(!base)) {
1547            error = -EFAULT;
1548            break;
1549        }
1550
1551        if (unlikely(!access_ok(VERIFY_WRITE, base, len))) {
1552            error = -EFAULT;
1553            break;
1554        }
1555
1556        sd.len = 0;
1557        sd.total_len = len;
1558        sd.flags = flags;
1559        sd.u.userptr = base;
1560        sd.pos = 0;
1561
1562        size = __splice_from_pipe(pipe, &sd, pipe_to_user);
1563        if (size < 0) {
1564            if (!ret)
1565                ret = size;
1566
1567            break;
1568        }
1569
1570        ret += size;
1571
1572        if (size < len)
1573            break;
1574
1575        nr_segs--;
1576        iov++;
1577    }
1578
1579    pipe_unlock(pipe);
1580
1581    if (!ret)
1582        ret = error;
1583
1584    return ret;
1585}
1586
1587/*
1588 * vmsplice splices a user address range into a pipe. It can be thought of
1589 * as splice-from-memory, where the regular splice is splice-from-file (or
1590 * to file). In both cases the output is a pipe, naturally.
1591 */
1592static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
1593                 unsigned long nr_segs, unsigned int flags)
1594{
1595    struct pipe_inode_info *pipe;
1596    struct page *pages[PIPE_BUFFERS];
1597    struct partial_page partial[PIPE_BUFFERS];
1598    struct splice_pipe_desc spd = {
1599        .pages = pages,
1600        .partial = partial,
1601        .flags = flags,
1602        .ops = &user_page_pipe_buf_ops,
1603        .spd_release = spd_release_page,
1604    };
1605
1606    pipe = pipe_info(file->f_path.dentry->d_inode);
1607    if (!pipe)
1608        return -EBADF;
1609
1610    spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial,
1611                        flags & SPLICE_F_GIFT);
1612    if (spd.nr_pages <= 0)
1613        return spd.nr_pages;
1614
1615    return splice_to_pipe(pipe, &spd);
1616}
1617
1618/*
1619 * Note that vmsplice only really supports true splicing _from_ user memory
1620 * to a pipe, not the other way around. Splicing from user memory is a simple
1621 * operation that can be supported without any funky alignment restrictions
1622 * or nasty vm tricks. We simply map in the user memory and fill them into
1623 * a pipe. The reverse isn't quite as easy, though. There are two possible
1624 * solutions for that:
1625 *
1626 * - memcpy() the data internally, at which point we might as well just
1627 * do a regular read() on the buffer anyway.
1628 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1629 * has restriction limitations on both ends of the pipe).
1630 *
1631 * Currently we punt and implement it as a normal copy, see pipe_to_user().
1632 *
1633 */
1634SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
1635        unsigned long, nr_segs, unsigned int, flags)
1636{
1637    struct file *file;
1638    long error;
1639    int fput;
1640
1641    if (unlikely(nr_segs > UIO_MAXIOV))
1642        return -EINVAL;
1643    else if (unlikely(!nr_segs))
1644        return 0;
1645
1646    error = -EBADF;
1647    file = fget_light(fd, &fput);
1648    if (file) {
1649        if (file->f_mode & FMODE_WRITE)
1650            error = vmsplice_to_pipe(file, iov, nr_segs, flags);
1651        else if (file->f_mode & FMODE_READ)
1652            error = vmsplice_to_user(file, iov, nr_segs, flags);
1653
1654        fput_light(file, fput);
1655    }
1656
1657    return error;
1658}
1659
1660SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1661        int, fd_out, loff_t __user *, off_out,
1662        size_t, len, unsigned int, flags)
1663{
1664    long error;
1665    struct file *in, *out;
1666    int fput_in, fput_out;
1667
1668    if (unlikely(!len))
1669        return 0;
1670
1671    error = -EBADF;
1672    in = fget_light(fd_in, &fput_in);
1673    if (in) {
1674        if (in->f_mode & FMODE_READ) {
1675            out = fget_light(fd_out, &fput_out);
1676            if (out) {
1677                if (out->f_mode & FMODE_WRITE)
1678                    error = do_splice(in, off_in,
1679                              out, off_out,
1680                              len, flags);
1681                fput_light(out, fput_out);
1682            }
1683        }
1684
1685        fput_light(in, fput_in);
1686    }
1687
1688    return error;
1689}
1690
1691/*
1692 * Make sure there's data to read. Wait for input if we can, otherwise
1693 * return an appropriate error.
1694 */
1695static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1696{
1697    int ret;
1698
1699    /*
1700     * Check ->nrbufs without the inode lock first. This function
1701     * is speculative anyways, so missing one is ok.
1702     */
1703    if (pipe->nrbufs)
1704        return 0;
1705
1706    ret = 0;
1707    pipe_lock(pipe);
1708
1709    while (!pipe->nrbufs) {
1710        if (signal_pending(current)) {
1711            ret = -ERESTARTSYS;
1712            break;
1713        }
1714        if (!pipe->writers)
1715            break;
1716        if (!pipe->waiting_writers) {
1717            if (flags & SPLICE_F_NONBLOCK) {
1718                ret = -EAGAIN;
1719                break;
1720            }
1721        }
1722        pipe_wait(pipe);
1723    }
1724
1725    pipe_unlock(pipe);
1726    return ret;
1727}
1728
1729/*
1730 * Make sure there's writeable room. Wait for room if we can, otherwise
1731 * return an appropriate error.
1732 */
1733static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1734{
1735    int ret;
1736
1737    /*
1738     * Check ->nrbufs without the inode lock first. This function
1739     * is speculative anyways, so missing one is ok.
1740     */
1741    if (pipe->nrbufs < PIPE_BUFFERS)
1742        return 0;
1743
1744    ret = 0;
1745    pipe_lock(pipe);
1746
1747    while (pipe->nrbufs >= PIPE_BUFFERS) {
1748        if (!pipe->readers) {
1749            send_sig(SIGPIPE, current, 0);
1750            ret = -EPIPE;
1751            break;
1752        }
1753        if (flags & SPLICE_F_NONBLOCK) {
1754            ret = -EAGAIN;
1755            break;
1756        }
1757        if (signal_pending(current)) {
1758            ret = -ERESTARTSYS;
1759            break;
1760        }
1761        pipe->waiting_writers++;
1762        pipe_wait(pipe);
1763        pipe->waiting_writers--;
1764    }
1765
1766    pipe_unlock(pipe);
1767    return ret;
1768}
1769
1770/*
1771 * Splice contents of ipipe to opipe.
1772 */
1773static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1774                   struct pipe_inode_info *opipe,
1775                   size_t len, unsigned int flags)
1776{
1777    struct pipe_buffer *ibuf, *obuf;
1778    int ret = 0, nbuf;
1779    bool input_wakeup = false;
1780
1781
1782retry:
1783    ret = ipipe_prep(ipipe, flags);
1784    if (ret)
1785        return ret;
1786
1787    ret = opipe_prep(opipe, flags);
1788    if (ret)
1789        return ret;
1790
1791    /*
1792     * Potential ABBA deadlock, work around it by ordering lock
1793     * grabbing by pipe info address. Otherwise two different processes
1794     * could deadlock (one doing tee from A -> B, the other from B -> A).
1795     */
1796    pipe_double_lock(ipipe, opipe);
1797
1798    do {
1799        if (!opipe->readers) {
1800            send_sig(SIGPIPE, current, 0);
1801            if (!ret)
1802                ret = -EPIPE;
1803            break;
1804        }
1805
1806        if (!ipipe->nrbufs && !ipipe->writers)
1807            break;
1808
1809        /*
1810         * Cannot make any progress, because either the input
1811         * pipe is empty or the output pipe is full.
1812         */
1813        if (!ipipe->nrbufs || opipe->nrbufs >= PIPE_BUFFERS) {
1814            /* Already processed some buffers, break */
1815            if (ret)
1816                break;
1817
1818            if (flags & SPLICE_F_NONBLOCK) {
1819                ret = -EAGAIN;
1820                break;
1821            }
1822
1823            /*
1824             * We raced with another reader/writer and haven't
1825             * managed to process any buffers. A zero return
1826             * value means EOF, so retry instead.
1827             */
1828            pipe_unlock(ipipe);
1829            pipe_unlock(opipe);
1830            goto retry;
1831        }
1832
1833        ibuf = ipipe->bufs + ipipe->curbuf;
1834        nbuf = (opipe->curbuf + opipe->nrbufs) % PIPE_BUFFERS;
1835        obuf = opipe->bufs + nbuf;
1836
1837        if (len >= ibuf->len) {
1838            /*
1839             * Simply move the whole buffer from ipipe to opipe
1840             */
1841            *obuf = *ibuf;
1842            ibuf->ops = NULL;
1843            opipe->nrbufs++;
1844            ipipe->curbuf = (ipipe->curbuf + 1) % PIPE_BUFFERS;
1845            ipipe->nrbufs--;
1846            input_wakeup = true;
1847        } else {
1848            /*
1849             * Get a reference to this pipe buffer,
1850             * so we can copy the contents over.
1851             */
1852            ibuf->ops->get(ipipe, ibuf);
1853            *obuf = *ibuf;
1854
1855            /*
1856             * Don't inherit the gift flag, we need to
1857             * prevent multiple steals of this page.
1858             */
1859            obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1860
1861            obuf->len = len;
1862            opipe->nrbufs++;
1863            ibuf->offset += obuf->len;
1864            ibuf->len -= obuf->len;
1865        }
1866        ret += obuf->len;
1867        len -= obuf->len;
1868    } while (len);
1869
1870    pipe_unlock(ipipe);
1871    pipe_unlock(opipe);
1872
1873    /*
1874     * If we put data in the output pipe, wakeup any potential readers.
1875     */
1876    if (ret > 0) {
1877        smp_mb();
1878        if (waitqueue_active(&opipe->wait))
1879            wake_up_interruptible(&opipe->wait);
1880        kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1881    }
1882    if (input_wakeup)
1883        wakeup_pipe_writers(ipipe);
1884
1885    return ret;
1886}
1887
1888/*
1889 * Link contents of ipipe to opipe.
1890 */
1891static int link_pipe(struct pipe_inode_info *ipipe,
1892             struct pipe_inode_info *opipe,
1893             size_t len, unsigned int flags)
1894{
1895    struct pipe_buffer *ibuf, *obuf;
1896    int ret = 0, i = 0, nbuf;
1897
1898    /*
1899     * Potential ABBA deadlock, work around it by ordering lock
1900     * grabbing by pipe info address. Otherwise two different processes
1901     * could deadlock (one doing tee from A -> B, the other from B -> A).
1902     */
1903    pipe_double_lock(ipipe, opipe);
1904
1905    do {
1906        if (!opipe->readers) {
1907            send_sig(SIGPIPE, current, 0);
1908            if (!ret)
1909                ret = -EPIPE;
1910            break;
1911        }
1912
1913        /*
1914         * If we have iterated all input buffers or ran out of
1915         * output room, break.
1916         */
1917        if (i >= ipipe->nrbufs || opipe->nrbufs >= PIPE_BUFFERS)
1918            break;
1919
1920        ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1921        nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1922
1923        /*
1924         * Get a reference to this pipe buffer,
1925         * so we can copy the contents over.
1926         */
1927        ibuf->ops->get(ipipe, ibuf);
1928
1929        obuf = opipe->bufs + nbuf;
1930        *obuf = *ibuf;
1931
1932        /*
1933         * Don't inherit the gift flag, we need to
1934         * prevent multiple steals of this page.
1935         */
1936        obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1937
1938        if (obuf->len > len)
1939            obuf->len = len;
1940
1941        opipe->nrbufs++;
1942        ret += obuf->len;
1943        len -= obuf->len;
1944        i++;
1945    } while (len);
1946
1947    /*
1948     * return EAGAIN if we have the potential of some data in the
1949     * future, otherwise just return 0
1950     */
1951    if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
1952        ret = -EAGAIN;
1953
1954    pipe_unlock(ipipe);
1955    pipe_unlock(opipe);
1956
1957    /*
1958     * If we put data in the output pipe, wakeup any potential readers.
1959     */
1960    if (ret > 0) {
1961        smp_mb();
1962        if (waitqueue_active(&opipe->wait))
1963            wake_up_interruptible(&opipe->wait);
1964        kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1965    }
1966
1967    return ret;
1968}
1969
1970/*
1971 * This is a tee(1) implementation that works on pipes. It doesn't copy
1972 * any data, it simply references the 'in' pages on the 'out' pipe.
1973 * The 'flags' used are the SPLICE_F_* variants, currently the only
1974 * applicable one is SPLICE_F_NONBLOCK.
1975 */
1976static long do_tee(struct file *in, struct file *out, size_t len,
1977           unsigned int flags)
1978{
1979    struct pipe_inode_info *ipipe = pipe_info(in->f_path.dentry->d_inode);
1980    struct pipe_inode_info *opipe = pipe_info(out->f_path.dentry->d_inode);
1981    int ret = -EINVAL;
1982
1983    /*
1984     * Duplicate the contents of ipipe to opipe without actually
1985     * copying the data.
1986     */
1987    if (ipipe && opipe && ipipe != opipe) {
1988        /*
1989         * Keep going, unless we encounter an error. The ipipe/opipe
1990         * ordering doesn't really matter.
1991         */
1992        ret = ipipe_prep(ipipe, flags);
1993        if (!ret) {
1994            ret = opipe_prep(opipe, flags);
1995            if (!ret)
1996                ret = link_pipe(ipipe, opipe, len, flags);
1997        }
1998    }
1999
2000    return ret;
2001}
2002
2003SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
2004{
2005    struct file *in;
2006    int error, fput_in;
2007
2008    if (unlikely(!len))
2009        return 0;
2010
2011    error = -EBADF;
2012    in = fget_light(fdin, &fput_in);
2013    if (in) {
2014        if (in->f_mode & FMODE_READ) {
2015            int fput_out;
2016            struct file *out = fget_light(fdout, &fput_out);
2017
2018            if (out) {
2019                if (out->f_mode & FMODE_WRITE)
2020                    error = do_tee(in, out, len, flags);
2021                fput_light(out, fput_out);
2022            }
2023        }
2024         fput_light(in, fput_in);
2025     }
2026
2027    return error;
2028}
2029

Archive Download this file



interactive