Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * linux/mm/madvise.c
4 : *
5 : * Copyright (C) 1999 Linus Torvalds
6 : * Copyright (C) 2002 Christoph Hellwig
7 : */
8 :
9 : #include <linux/mman.h>
10 : #include <linux/pagemap.h>
11 : #include <linux/syscalls.h>
12 : #include <linux/mempolicy.h>
13 : #include <linux/page-isolation.h>
14 : #include <linux/page_idle.h>
15 : #include <linux/userfaultfd_k.h>
16 : #include <linux/hugetlb.h>
17 : #include <linux/falloc.h>
18 : #include <linux/fadvise.h>
19 : #include <linux/sched.h>
20 : #include <linux/sched/mm.h>
21 : #include <linux/mm_inline.h>
22 : #include <linux/string.h>
23 : #include <linux/uio.h>
24 : #include <linux/ksm.h>
25 : #include <linux/fs.h>
26 : #include <linux/file.h>
27 : #include <linux/blkdev.h>
28 : #include <linux/backing-dev.h>
29 : #include <linux/pagewalk.h>
30 : #include <linux/swap.h>
31 : #include <linux/swapops.h>
32 : #include <linux/shmem_fs.h>
33 : #include <linux/mmu_notifier.h>
34 :
35 : #include <asm/tlb.h>
36 :
37 : #include "internal.h"
38 : #include "swap.h"
39 :
40 : struct madvise_walk_private {
41 : struct mmu_gather *tlb;
42 : bool pageout;
43 : };
44 :
45 : /*
46 : * Any behaviour which results in changes to the vma->vm_flags needs to
47 : * take mmap_lock for writing. Others, which simply traverse vmas, need
48 : * to only take it for reading.
49 : */
50 : static int madvise_need_mmap_write(int behavior)
51 : {
52 : switch (behavior) {
53 : case MADV_REMOVE:
54 : case MADV_WILLNEED:
55 : case MADV_DONTNEED:
56 : case MADV_DONTNEED_LOCKED:
57 : case MADV_COLD:
58 : case MADV_PAGEOUT:
59 : case MADV_FREE:
60 : case MADV_POPULATE_READ:
61 : case MADV_POPULATE_WRITE:
62 : case MADV_COLLAPSE:
63 : return 0;
64 : default:
65 : /* be safe, default to 1. list exceptions explicitly */
66 : return 1;
67 : }
68 : }
69 :
70 : #ifdef CONFIG_ANON_VMA_NAME
71 : struct anon_vma_name *anon_vma_name_alloc(const char *name)
72 : {
73 : struct anon_vma_name *anon_name;
74 : size_t count;
75 :
76 : /* Add 1 for NUL terminator at the end of the anon_name->name */
77 : count = strlen(name) + 1;
78 : anon_name = kmalloc(struct_size(anon_name, name, count), GFP_KERNEL);
79 : if (anon_name) {
80 : kref_init(&anon_name->kref);
81 : memcpy(anon_name->name, name, count);
82 : }
83 :
84 : return anon_name;
85 : }
86 :
87 : void anon_vma_name_free(struct kref *kref)
88 : {
89 : struct anon_vma_name *anon_name =
90 : container_of(kref, struct anon_vma_name, kref);
91 : kfree(anon_name);
92 : }
93 :
94 : struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
95 : {
96 : mmap_assert_locked(vma->vm_mm);
97 :
98 : return vma->anon_name;
99 : }
100 :
101 : /* mmap_lock should be write-locked */
102 : static int replace_anon_vma_name(struct vm_area_struct *vma,
103 : struct anon_vma_name *anon_name)
104 : {
105 : struct anon_vma_name *orig_name = anon_vma_name(vma);
106 :
107 : if (!anon_name) {
108 : vma->anon_name = NULL;
109 : anon_vma_name_put(orig_name);
110 : return 0;
111 : }
112 :
113 : if (anon_vma_name_eq(orig_name, anon_name))
114 : return 0;
115 :
116 : vma->anon_name = anon_vma_name_reuse(anon_name);
117 : anon_vma_name_put(orig_name);
118 :
119 : return 0;
120 : }
121 : #else /* CONFIG_ANON_VMA_NAME */
122 : static int replace_anon_vma_name(struct vm_area_struct *vma,
123 : struct anon_vma_name *anon_name)
124 : {
125 0 : if (anon_name)
126 : return -EINVAL;
127 :
128 : return 0;
129 : }
130 : #endif /* CONFIG_ANON_VMA_NAME */
131 : /*
132 : * Update the vm_flags on region of a vma, splitting it or merging it as
133 : * necessary. Must be called with mmap_lock held for writing;
134 : * Caller should ensure anon_name stability by raising its refcount even when
135 : * anon_name belongs to a valid vma because this function might free that vma.
136 : */
137 0 : static int madvise_update_vma(struct vm_area_struct *vma,
138 : struct vm_area_struct **prev, unsigned long start,
139 : unsigned long end, unsigned long new_flags,
140 : struct anon_vma_name *anon_name)
141 : {
142 0 : struct mm_struct *mm = vma->vm_mm;
143 : int error;
144 : pgoff_t pgoff;
145 0 : VMA_ITERATOR(vmi, mm, start);
146 :
147 0 : if (new_flags == vma->vm_flags && anon_vma_name_eq(anon_vma_name(vma), anon_name)) {
148 0 : *prev = vma;
149 0 : return 0;
150 : }
151 :
152 0 : pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
153 0 : *prev = vma_merge(&vmi, mm, *prev, start, end, new_flags,
154 : vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
155 : vma->vm_userfaultfd_ctx, anon_name);
156 0 : if (*prev) {
157 : vma = *prev;
158 : goto success;
159 : }
160 :
161 0 : *prev = vma;
162 :
163 0 : if (start != vma->vm_start) {
164 0 : error = split_vma(&vmi, vma, start, 1);
165 0 : if (error)
166 : return error;
167 : }
168 :
169 0 : if (end != vma->vm_end) {
170 0 : error = split_vma(&vmi, vma, end, 0);
171 0 : if (error)
172 : return error;
173 : }
174 :
175 : success:
176 : /*
177 : * vm_flags is protected by the mmap_lock held in write mode.
178 : */
179 0 : vm_flags_reset(vma, new_flags);
180 0 : if (!vma->vm_file || vma_is_anon_shmem(vma)) {
181 0 : error = replace_anon_vma_name(vma, anon_name);
182 0 : if (error)
183 : return error;
184 : }
185 :
186 : return 0;
187 : }
188 :
189 : #ifdef CONFIG_SWAP
190 0 : static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
191 : unsigned long end, struct mm_walk *walk)
192 : {
193 0 : struct vm_area_struct *vma = walk->private;
194 : unsigned long index;
195 0 : struct swap_iocb *splug = NULL;
196 :
197 0 : if (pmd_none_or_trans_huge_or_clear_bad(pmd))
198 : return 0;
199 :
200 0 : for (index = start; index != end; index += PAGE_SIZE) {
201 : pte_t pte;
202 : swp_entry_t entry;
203 : struct page *page;
204 : spinlock_t *ptl;
205 : pte_t *ptep;
206 :
207 0 : ptep = pte_offset_map_lock(vma->vm_mm, pmd, index, &ptl);
208 0 : pte = *ptep;
209 0 : pte_unmap_unlock(ptep, ptl);
210 :
211 0 : if (!is_swap_pte(pte))
212 0 : continue;
213 0 : entry = pte_to_swp_entry(pte);
214 0 : if (unlikely(non_swap_entry(entry)))
215 0 : continue;
216 :
217 0 : page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
218 : vma, index, false, &splug);
219 0 : if (page)
220 0 : put_page(page);
221 : }
222 0 : swap_read_unplug(splug);
223 0 : cond_resched();
224 :
225 0 : return 0;
226 : }
227 :
228 : static const struct mm_walk_ops swapin_walk_ops = {
229 : .pmd_entry = swapin_walk_pmd_entry,
230 : };
231 :
232 0 : static void force_shm_swapin_readahead(struct vm_area_struct *vma,
233 : unsigned long start, unsigned long end,
234 : struct address_space *mapping)
235 : {
236 0 : XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
237 0 : pgoff_t end_index = linear_page_index(vma, end + PAGE_SIZE - 1);
238 : struct page *page;
239 0 : struct swap_iocb *splug = NULL;
240 :
241 : rcu_read_lock();
242 0 : xas_for_each(&xas, page, end_index) {
243 : swp_entry_t swap;
244 :
245 0 : if (!xa_is_value(page))
246 0 : continue;
247 0 : swap = radix_to_swp_entry(page);
248 : /* There might be swapin error entries in shmem mapping. */
249 0 : if (non_swap_entry(swap))
250 0 : continue;
251 0 : xas_pause(&xas);
252 : rcu_read_unlock();
253 :
254 0 : page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
255 : NULL, 0, false, &splug);
256 0 : if (page)
257 0 : put_page(page);
258 :
259 : rcu_read_lock();
260 : }
261 : rcu_read_unlock();
262 0 : swap_read_unplug(splug);
263 :
264 0 : lru_add_drain(); /* Push any new pages onto the LRU now */
265 0 : }
266 : #endif /* CONFIG_SWAP */
267 :
268 : /*
269 : * Schedule all required I/O operations. Do not wait for completion.
270 : */
271 0 : static long madvise_willneed(struct vm_area_struct *vma,
272 : struct vm_area_struct **prev,
273 : unsigned long start, unsigned long end)
274 : {
275 0 : struct mm_struct *mm = vma->vm_mm;
276 0 : struct file *file = vma->vm_file;
277 : loff_t offset;
278 :
279 0 : *prev = vma;
280 : #ifdef CONFIG_SWAP
281 0 : if (!file) {
282 0 : walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma);
283 0 : lru_add_drain(); /* Push any new pages onto the LRU now */
284 0 : return 0;
285 : }
286 :
287 0 : if (shmem_mapping(file->f_mapping)) {
288 0 : force_shm_swapin_readahead(vma, start, end,
289 : file->f_mapping);
290 0 : return 0;
291 : }
292 : #else
293 : if (!file)
294 : return -EBADF;
295 : #endif
296 :
297 0 : if (IS_DAX(file_inode(file))) {
298 : /* no bad return value, but ignore advice */
299 : return 0;
300 : }
301 :
302 : /*
303 : * Filesystem's fadvise may need to take various locks. We need to
304 : * explicitly grab a reference because the vma (and hence the
305 : * vma's reference to the file) can go away as soon as we drop
306 : * mmap_lock.
307 : */
308 0 : *prev = NULL; /* tell sys_madvise we drop mmap_lock */
309 0 : get_file(file);
310 0 : offset = (loff_t)(start - vma->vm_start)
311 0 : + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
312 0 : mmap_read_unlock(mm);
313 0 : vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
314 0 : fput(file);
315 0 : mmap_read_lock(mm);
316 0 : return 0;
317 : }
318 :
319 0 : static inline bool can_do_file_pageout(struct vm_area_struct *vma)
320 : {
321 0 : if (!vma->vm_file)
322 : return false;
323 : /*
324 : * paging out pagecache only for non-anonymous mappings that correspond
325 : * to the files the calling process could (if tried) open for writing;
326 : * otherwise we'd be including shared non-exclusive mappings, which
327 : * opens a side channel.
328 : */
329 0 : return inode_owner_or_capable(&nop_mnt_idmap,
330 0 : file_inode(vma->vm_file)) ||
331 0 : file_permission(vma->vm_file, MAY_WRITE) == 0;
332 : }
333 :
334 0 : static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
335 : unsigned long addr, unsigned long end,
336 : struct mm_walk *walk)
337 : {
338 0 : struct madvise_walk_private *private = walk->private;
339 0 : struct mmu_gather *tlb = private->tlb;
340 0 : bool pageout = private->pageout;
341 0 : struct mm_struct *mm = tlb->mm;
342 0 : struct vm_area_struct *vma = walk->vma;
343 : pte_t *orig_pte, *pte, ptent;
344 : spinlock_t *ptl;
345 0 : struct folio *folio = NULL;
346 0 : LIST_HEAD(folio_list);
347 : bool pageout_anon_only_filter;
348 :
349 0 : if (fatal_signal_pending(current))
350 : return -EINTR;
351 :
352 0 : pageout_anon_only_filter = pageout && !vma_is_anonymous(vma) &&
353 0 : !can_do_file_pageout(vma);
354 :
355 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
356 : if (pmd_trans_huge(*pmd)) {
357 : pmd_t orig_pmd;
358 : unsigned long next = pmd_addr_end(addr, end);
359 :
360 : tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
361 : ptl = pmd_trans_huge_lock(pmd, vma);
362 : if (!ptl)
363 : return 0;
364 :
365 : orig_pmd = *pmd;
366 : if (is_huge_zero_pmd(orig_pmd))
367 : goto huge_unlock;
368 :
369 : if (unlikely(!pmd_present(orig_pmd))) {
370 : VM_BUG_ON(thp_migration_supported() &&
371 : !is_pmd_migration_entry(orig_pmd));
372 : goto huge_unlock;
373 : }
374 :
375 : folio = pfn_folio(pmd_pfn(orig_pmd));
376 :
377 : /* Do not interfere with other mappings of this folio */
378 : if (folio_mapcount(folio) != 1)
379 : goto huge_unlock;
380 :
381 : if (pageout_anon_only_filter && !folio_test_anon(folio))
382 : goto huge_unlock;
383 :
384 : if (next - addr != HPAGE_PMD_SIZE) {
385 : int err;
386 :
387 : folio_get(folio);
388 : spin_unlock(ptl);
389 : folio_lock(folio);
390 : err = split_folio(folio);
391 : folio_unlock(folio);
392 : folio_put(folio);
393 : if (!err)
394 : goto regular_folio;
395 : return 0;
396 : }
397 :
398 : if (pmd_young(orig_pmd)) {
399 : pmdp_invalidate(vma, addr, pmd);
400 : orig_pmd = pmd_mkold(orig_pmd);
401 :
402 : set_pmd_at(mm, addr, pmd, orig_pmd);
403 : tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
404 : }
405 :
406 : folio_clear_referenced(folio);
407 : folio_test_clear_young(folio);
408 : if (pageout) {
409 : if (folio_isolate_lru(folio)) {
410 : if (folio_test_unevictable(folio))
411 : folio_putback_lru(folio);
412 : else
413 : list_add(&folio->lru, &folio_list);
414 : }
415 : } else
416 : folio_deactivate(folio);
417 : huge_unlock:
418 : spin_unlock(ptl);
419 : if (pageout)
420 : reclaim_pages(&folio_list);
421 : return 0;
422 : }
423 :
424 : regular_folio:
425 : if (pmd_trans_unstable(pmd))
426 : return 0;
427 : #endif
428 0 : tlb_change_page_size(tlb, PAGE_SIZE);
429 0 : orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
430 0 : flush_tlb_batched_pending(mm);
431 : arch_enter_lazy_mmu_mode();
432 0 : for (; addr < end; pte++, addr += PAGE_SIZE) {
433 0 : ptent = *pte;
434 :
435 0 : if (pte_none(ptent))
436 0 : continue;
437 :
438 0 : if (!pte_present(ptent))
439 0 : continue;
440 :
441 0 : folio = vm_normal_folio(vma, addr, ptent);
442 0 : if (!folio || folio_is_zone_device(folio))
443 0 : continue;
444 :
445 : /*
446 : * Creating a THP page is expensive so split it only if we
447 : * are sure it's worth. Split it if we are only owner.
448 : */
449 0 : if (folio_test_large(folio)) {
450 0 : if (folio_mapcount(folio) != 1)
451 : break;
452 0 : if (pageout_anon_only_filter && !folio_test_anon(folio))
453 : break;
454 0 : folio_get(folio);
455 0 : if (!folio_trylock(folio)) {
456 : folio_put(folio);
457 : break;
458 : }
459 0 : pte_unmap_unlock(orig_pte, ptl);
460 0 : if (split_folio(folio)) {
461 : folio_unlock(folio);
462 : folio_put(folio);
463 : orig_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
464 : break;
465 : }
466 0 : folio_unlock(folio);
467 0 : folio_put(folio);
468 0 : orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
469 0 : pte--;
470 0 : addr -= PAGE_SIZE;
471 0 : continue;
472 : }
473 :
474 : /*
475 : * Do not interfere with other mappings of this folio and
476 : * non-LRU folio.
477 : */
478 0 : if (!folio_test_lru(folio) || folio_mapcount(folio) != 1)
479 0 : continue;
480 :
481 0 : if (pageout_anon_only_filter && !folio_test_anon(folio))
482 0 : continue;
483 :
484 : VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
485 :
486 0 : if (pte_young(ptent)) {
487 0 : ptent = ptep_get_and_clear_full(mm, addr, pte,
488 0 : tlb->fullmm);
489 0 : ptent = pte_mkold(ptent);
490 0 : set_pte_at(mm, addr, pte, ptent);
491 : tlb_remove_tlb_entry(tlb, pte, addr);
492 : }
493 :
494 : /*
495 : * We are deactivating a folio for accelerating reclaiming.
496 : * VM couldn't reclaim the folio unless we clear PG_young.
497 : * As a side effect, it makes confuse idle-page tracking
498 : * because they will miss recent referenced history.
499 : */
500 0 : folio_clear_referenced(folio);
501 0 : folio_test_clear_young(folio);
502 0 : if (pageout) {
503 0 : if (folio_isolate_lru(folio)) {
504 0 : if (folio_test_unevictable(folio))
505 0 : folio_putback_lru(folio);
506 : else
507 0 : list_add(&folio->lru, &folio_list);
508 : }
509 : } else
510 0 : folio_deactivate(folio);
511 : }
512 :
513 : arch_leave_lazy_mmu_mode();
514 0 : pte_unmap_unlock(orig_pte, ptl);
515 0 : if (pageout)
516 0 : reclaim_pages(&folio_list);
517 0 : cond_resched();
518 :
519 0 : return 0;
520 : }
521 :
522 : static const struct mm_walk_ops cold_walk_ops = {
523 : .pmd_entry = madvise_cold_or_pageout_pte_range,
524 : };
525 :
526 0 : static void madvise_cold_page_range(struct mmu_gather *tlb,
527 : struct vm_area_struct *vma,
528 : unsigned long addr, unsigned long end)
529 : {
530 0 : struct madvise_walk_private walk_private = {
531 : .pageout = false,
532 : .tlb = tlb,
533 : };
534 :
535 0 : tlb_start_vma(tlb, vma);
536 0 : walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
537 0 : tlb_end_vma(tlb, vma);
538 0 : }
539 :
540 : static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
541 : {
542 0 : return !(vma->vm_flags & (VM_LOCKED|VM_PFNMAP|VM_HUGETLB));
543 : }
544 :
545 0 : static long madvise_cold(struct vm_area_struct *vma,
546 : struct vm_area_struct **prev,
547 : unsigned long start_addr, unsigned long end_addr)
548 : {
549 0 : struct mm_struct *mm = vma->vm_mm;
550 : struct mmu_gather tlb;
551 :
552 0 : *prev = vma;
553 0 : if (!can_madv_lru_vma(vma))
554 : return -EINVAL;
555 :
556 0 : lru_add_drain();
557 0 : tlb_gather_mmu(&tlb, mm);
558 0 : madvise_cold_page_range(&tlb, vma, start_addr, end_addr);
559 0 : tlb_finish_mmu(&tlb);
560 :
561 0 : return 0;
562 : }
563 :
564 0 : static void madvise_pageout_page_range(struct mmu_gather *tlb,
565 : struct vm_area_struct *vma,
566 : unsigned long addr, unsigned long end)
567 : {
568 0 : struct madvise_walk_private walk_private = {
569 : .pageout = true,
570 : .tlb = tlb,
571 : };
572 :
573 0 : tlb_start_vma(tlb, vma);
574 0 : walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, &walk_private);
575 0 : tlb_end_vma(tlb, vma);
576 0 : }
577 :
578 0 : static long madvise_pageout(struct vm_area_struct *vma,
579 : struct vm_area_struct **prev,
580 : unsigned long start_addr, unsigned long end_addr)
581 : {
582 0 : struct mm_struct *mm = vma->vm_mm;
583 : struct mmu_gather tlb;
584 :
585 0 : *prev = vma;
586 0 : if (!can_madv_lru_vma(vma))
587 : return -EINVAL;
588 :
589 : /*
590 : * If the VMA belongs to a private file mapping, there can be private
591 : * dirty pages which can be paged out if even this process is neither
592 : * owner nor write capable of the file. We allow private file mappings
593 : * further to pageout dirty anon pages.
594 : */
595 0 : if (!vma_is_anonymous(vma) && (!can_do_file_pageout(vma) &&
596 0 : (vma->vm_flags & VM_MAYSHARE)))
597 : return 0;
598 :
599 0 : lru_add_drain();
600 0 : tlb_gather_mmu(&tlb, mm);
601 0 : madvise_pageout_page_range(&tlb, vma, start_addr, end_addr);
602 0 : tlb_finish_mmu(&tlb);
603 :
604 0 : return 0;
605 : }
606 :
607 0 : static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
608 : unsigned long end, struct mm_walk *walk)
609 :
610 : {
611 0 : struct mmu_gather *tlb = walk->private;
612 0 : struct mm_struct *mm = tlb->mm;
613 0 : struct vm_area_struct *vma = walk->vma;
614 : spinlock_t *ptl;
615 : pte_t *orig_pte, *pte, ptent;
616 : struct folio *folio;
617 0 : int nr_swap = 0;
618 : unsigned long next;
619 :
620 0 : next = pmd_addr_end(addr, end);
621 0 : if (pmd_trans_huge(*pmd))
622 : if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next))
623 : goto next;
624 :
625 0 : if (pmd_trans_unstable(pmd))
626 : return 0;
627 :
628 0 : tlb_change_page_size(tlb, PAGE_SIZE);
629 0 : orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
630 0 : flush_tlb_batched_pending(mm);
631 : arch_enter_lazy_mmu_mode();
632 0 : for (; addr != end; pte++, addr += PAGE_SIZE) {
633 0 : ptent = *pte;
634 :
635 0 : if (pte_none(ptent))
636 0 : continue;
637 : /*
638 : * If the pte has swp_entry, just clear page table to
639 : * prevent swap-in which is more expensive rather than
640 : * (page allocation + zeroing).
641 : */
642 0 : if (!pte_present(ptent)) {
643 : swp_entry_t entry;
644 :
645 0 : entry = pte_to_swp_entry(ptent);
646 0 : if (!non_swap_entry(entry)) {
647 0 : nr_swap--;
648 0 : free_swap_and_cache(entry);
649 0 : pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
650 0 : } else if (is_hwpoison_entry(entry) ||
651 0 : is_swapin_error_entry(entry)) {
652 0 : pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
653 : }
654 0 : continue;
655 : }
656 :
657 0 : folio = vm_normal_folio(vma, addr, ptent);
658 0 : if (!folio || folio_is_zone_device(folio))
659 0 : continue;
660 :
661 : /*
662 : * If pmd isn't transhuge but the folio is large and
663 : * is owned by only this process, split it and
664 : * deactivate all pages.
665 : */
666 0 : if (folio_test_large(folio)) {
667 0 : if (folio_mapcount(folio) != 1)
668 : goto out;
669 0 : folio_get(folio);
670 0 : if (!folio_trylock(folio)) {
671 : folio_put(folio);
672 : goto out;
673 : }
674 0 : pte_unmap_unlock(orig_pte, ptl);
675 0 : if (split_folio(folio)) {
676 : folio_unlock(folio);
677 : folio_put(folio);
678 : orig_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
679 : goto out;
680 : }
681 0 : folio_unlock(folio);
682 0 : folio_put(folio);
683 0 : orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
684 0 : pte--;
685 0 : addr -= PAGE_SIZE;
686 0 : continue;
687 : }
688 :
689 0 : if (folio_test_swapcache(folio) || folio_test_dirty(folio)) {
690 0 : if (!folio_trylock(folio))
691 0 : continue;
692 : /*
693 : * If folio is shared with others, we mustn't clear
694 : * the folio's dirty flag.
695 : */
696 0 : if (folio_mapcount(folio) != 1) {
697 0 : folio_unlock(folio);
698 0 : continue;
699 : }
700 :
701 0 : if (folio_test_swapcache(folio) &&
702 0 : !folio_free_swap(folio)) {
703 0 : folio_unlock(folio);
704 0 : continue;
705 : }
706 :
707 0 : folio_clear_dirty(folio);
708 0 : folio_unlock(folio);
709 : }
710 :
711 0 : if (pte_young(ptent) || pte_dirty(ptent)) {
712 : /*
713 : * Some of architecture(ex, PPC) don't update TLB
714 : * with set_pte_at and tlb_remove_tlb_entry so for
715 : * the portability, remap the pte with old|clean
716 : * after pte clearing.
717 : */
718 0 : ptent = ptep_get_and_clear_full(mm, addr, pte,
719 0 : tlb->fullmm);
720 :
721 0 : ptent = pte_mkold(ptent);
722 0 : ptent = pte_mkclean(ptent);
723 0 : set_pte_at(mm, addr, pte, ptent);
724 : tlb_remove_tlb_entry(tlb, pte, addr);
725 : }
726 0 : folio_mark_lazyfree(folio);
727 : }
728 : out:
729 0 : if (nr_swap) {
730 0 : if (current->mm == mm)
731 : sync_mm_rss(mm);
732 :
733 0 : add_mm_counter(mm, MM_SWAPENTS, nr_swap);
734 : }
735 : arch_leave_lazy_mmu_mode();
736 0 : pte_unmap_unlock(orig_pte, ptl);
737 0 : cond_resched();
738 : next:
739 : return 0;
740 : }
741 :
742 : static const struct mm_walk_ops madvise_free_walk_ops = {
743 : .pmd_entry = madvise_free_pte_range,
744 : };
745 :
746 0 : static int madvise_free_single_vma(struct vm_area_struct *vma,
747 : unsigned long start_addr, unsigned long end_addr)
748 : {
749 0 : struct mm_struct *mm = vma->vm_mm;
750 : struct mmu_notifier_range range;
751 : struct mmu_gather tlb;
752 :
753 : /* MADV_FREE works for only anon vma at the moment */
754 0 : if (!vma_is_anonymous(vma))
755 : return -EINVAL;
756 :
757 0 : range.start = max(vma->vm_start, start_addr);
758 0 : if (range.start >= vma->vm_end)
759 : return -EINVAL;
760 0 : range.end = min(vma->vm_end, end_addr);
761 0 : if (range.end <= vma->vm_start)
762 : return -EINVAL;
763 0 : mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
764 : range.start, range.end);
765 :
766 0 : lru_add_drain();
767 0 : tlb_gather_mmu(&tlb, mm);
768 0 : update_hiwater_rss(mm);
769 :
770 0 : mmu_notifier_invalidate_range_start(&range);
771 0 : tlb_start_vma(&tlb, vma);
772 0 : walk_page_range(vma->vm_mm, range.start, range.end,
773 : &madvise_free_walk_ops, &tlb);
774 0 : tlb_end_vma(&tlb, vma);
775 0 : mmu_notifier_invalidate_range_end(&range);
776 0 : tlb_finish_mmu(&tlb);
777 :
778 0 : return 0;
779 : }
780 :
781 : /*
782 : * Application no longer needs these pages. If the pages are dirty,
783 : * it's OK to just throw them away. The app will be more careful about
784 : * data it wants to keep. Be sure to free swap resources too. The
785 : * zap_page_range_single call sets things up for shrink_active_list to actually
786 : * free these pages later if no one else has touched them in the meantime,
787 : * although we could add these pages to a global reuse list for
788 : * shrink_active_list to pick up before reclaiming other pages.
789 : *
790 : * NB: This interface discards data rather than pushes it out to swap,
791 : * as some implementations do. This has performance implications for
792 : * applications like large transactional databases which want to discard
793 : * pages in anonymous maps after committing to backing store the data
794 : * that was kept in them. There is no reason to write this data out to
795 : * the swap area if the application is discarding it.
796 : *
797 : * An interface that causes the system to free clean pages and flush
798 : * dirty pages is already available as msync(MS_INVALIDATE).
799 : */
800 : static long madvise_dontneed_single_vma(struct vm_area_struct *vma,
801 : unsigned long start, unsigned long end)
802 : {
803 0 : zap_page_range_single(vma, start, end - start, NULL);
804 : return 0;
805 : }
806 :
807 : static bool madvise_dontneed_free_valid_vma(struct vm_area_struct *vma,
808 : unsigned long start,
809 : unsigned long *end,
810 : int behavior)
811 : {
812 0 : if (!is_vm_hugetlb_page(vma)) {
813 0 : unsigned int forbidden = VM_PFNMAP;
814 :
815 0 : if (behavior != MADV_DONTNEED_LOCKED)
816 0 : forbidden |= VM_LOCKED;
817 :
818 0 : return !(vma->vm_flags & forbidden);
819 : }
820 :
821 : if (behavior != MADV_DONTNEED && behavior != MADV_DONTNEED_LOCKED)
822 : return false;
823 : if (start & ~huge_page_mask(hstate_vma(vma)))
824 : return false;
825 :
826 : /*
827 : * Madvise callers expect the length to be rounded up to PAGE_SIZE
828 : * boundaries, and may be unaware that this VMA uses huge pages.
829 : * Avoid unexpected data loss by rounding down the number of
830 : * huge pages freed.
831 : */
832 : *end = ALIGN_DOWN(*end, huge_page_size(hstate_vma(vma)));
833 :
834 : return true;
835 : }
836 :
837 0 : static long madvise_dontneed_free(struct vm_area_struct *vma,
838 : struct vm_area_struct **prev,
839 : unsigned long start, unsigned long end,
840 : int behavior)
841 : {
842 0 : struct mm_struct *mm = vma->vm_mm;
843 :
844 0 : *prev = vma;
845 0 : if (!madvise_dontneed_free_valid_vma(vma, start, &end, behavior))
846 : return -EINVAL;
847 :
848 0 : if (start == end)
849 : return 0;
850 :
851 0 : if (!userfaultfd_remove(vma, start, end)) {
852 : *prev = NULL; /* mmap_lock has been dropped, prev is stale */
853 :
854 : mmap_read_lock(mm);
855 : vma = vma_lookup(mm, start);
856 : if (!vma)
857 : return -ENOMEM;
858 : /*
859 : * Potential end adjustment for hugetlb vma is OK as
860 : * the check below keeps end within vma.
861 : */
862 : if (!madvise_dontneed_free_valid_vma(vma, start, &end,
863 : behavior))
864 : return -EINVAL;
865 : if (end > vma->vm_end) {
866 : /*
867 : * Don't fail if end > vma->vm_end. If the old
868 : * vma was split while the mmap_lock was
869 : * released the effect of the concurrent
870 : * operation may not cause madvise() to
871 : * have an undefined result. There may be an
872 : * adjacent next vma that we'll walk
873 : * next. userfaultfd_remove() will generate an
874 : * UFFD_EVENT_REMOVE repetition on the
875 : * end-vma->vm_end range, but the manager can
876 : * handle a repetition fine.
877 : */
878 : end = vma->vm_end;
879 : }
880 : VM_WARN_ON(start >= end);
881 : }
882 :
883 0 : if (behavior == MADV_DONTNEED || behavior == MADV_DONTNEED_LOCKED)
884 0 : return madvise_dontneed_single_vma(vma, start, end);
885 0 : else if (behavior == MADV_FREE)
886 0 : return madvise_free_single_vma(vma, start, end);
887 : else
888 : return -EINVAL;
889 : }
890 :
891 0 : static long madvise_populate(struct vm_area_struct *vma,
892 : struct vm_area_struct **prev,
893 : unsigned long start, unsigned long end,
894 : int behavior)
895 : {
896 0 : const bool write = behavior == MADV_POPULATE_WRITE;
897 0 : struct mm_struct *mm = vma->vm_mm;
898 : unsigned long tmp_end;
899 0 : int locked = 1;
900 : long pages;
901 :
902 0 : *prev = vma;
903 :
904 0 : while (start < end) {
905 : /*
906 : * We might have temporarily dropped the lock. For example,
907 : * our VMA might have been split.
908 : */
909 0 : if (!vma || start >= vma->vm_end) {
910 0 : vma = vma_lookup(mm, start);
911 0 : if (!vma)
912 : return -ENOMEM;
913 : }
914 :
915 0 : tmp_end = min_t(unsigned long, end, vma->vm_end);
916 : /* Populate (prefault) page tables readable/writable. */
917 0 : pages = faultin_vma_page_range(vma, start, tmp_end, write,
918 : &locked);
919 0 : if (!locked) {
920 0 : mmap_read_lock(mm);
921 0 : locked = 1;
922 0 : *prev = NULL;
923 0 : vma = NULL;
924 : }
925 0 : if (pages < 0) {
926 0 : switch (pages) {
927 : case -EINTR:
928 : return -EINTR;
929 : case -EINVAL: /* Incompatible mappings / permissions. */
930 0 : return -EINVAL;
931 : case -EHWPOISON:
932 0 : return -EHWPOISON;
933 : case -EFAULT: /* VM_FAULT_SIGBUS or VM_FAULT_SIGSEGV */
934 0 : return -EFAULT;
935 : default:
936 0 : pr_warn_once("%s: unhandled return value: %ld\n",
937 : __func__, pages);
938 : fallthrough;
939 : case -ENOMEM:
940 : return -ENOMEM;
941 : }
942 : }
943 0 : start += pages * PAGE_SIZE;
944 : }
945 : return 0;
946 : }
947 :
948 : /*
949 : * Application wants to free up the pages and associated backing store.
950 : * This is effectively punching a hole into the middle of a file.
951 : */
952 0 : static long madvise_remove(struct vm_area_struct *vma,
953 : struct vm_area_struct **prev,
954 : unsigned long start, unsigned long end)
955 : {
956 : loff_t offset;
957 : int error;
958 : struct file *f;
959 0 : struct mm_struct *mm = vma->vm_mm;
960 :
961 0 : *prev = NULL; /* tell sys_madvise we drop mmap_lock */
962 :
963 0 : if (vma->vm_flags & VM_LOCKED)
964 : return -EINVAL;
965 :
966 0 : f = vma->vm_file;
967 :
968 0 : if (!f || !f->f_mapping || !f->f_mapping->host) {
969 : return -EINVAL;
970 : }
971 :
972 0 : if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
973 : return -EACCES;
974 :
975 0 : offset = (loff_t)(start - vma->vm_start)
976 0 : + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
977 :
978 : /*
979 : * Filesystem's fallocate may need to take i_rwsem. We need to
980 : * explicitly grab a reference because the vma (and hence the
981 : * vma's reference to the file) can go away as soon as we drop
982 : * mmap_lock.
983 : */
984 0 : get_file(f);
985 0 : if (userfaultfd_remove(vma, start, end)) {
986 : /* mmap_lock was not released by userfaultfd_remove() */
987 : mmap_read_unlock(mm);
988 : }
989 0 : error = vfs_fallocate(f,
990 : FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
991 0 : offset, end - start);
992 0 : fput(f);
993 0 : mmap_read_lock(mm);
994 0 : return error;
995 : }
996 :
997 : /*
998 : * Apply an madvise behavior to a region of a vma. madvise_update_vma
999 : * will handle splitting a vm area into separate areas, each area with its own
1000 : * behavior.
1001 : */
1002 0 : static int madvise_vma_behavior(struct vm_area_struct *vma,
1003 : struct vm_area_struct **prev,
1004 : unsigned long start, unsigned long end,
1005 : unsigned long behavior)
1006 : {
1007 : int error;
1008 : struct anon_vma_name *anon_name;
1009 0 : unsigned long new_flags = vma->vm_flags;
1010 :
1011 0 : switch (behavior) {
1012 : case MADV_REMOVE:
1013 0 : return madvise_remove(vma, prev, start, end);
1014 : case MADV_WILLNEED:
1015 0 : return madvise_willneed(vma, prev, start, end);
1016 : case MADV_COLD:
1017 0 : return madvise_cold(vma, prev, start, end);
1018 : case MADV_PAGEOUT:
1019 0 : return madvise_pageout(vma, prev, start, end);
1020 : case MADV_FREE:
1021 : case MADV_DONTNEED:
1022 : case MADV_DONTNEED_LOCKED:
1023 0 : return madvise_dontneed_free(vma, prev, start, end, behavior);
1024 : case MADV_POPULATE_READ:
1025 : case MADV_POPULATE_WRITE:
1026 0 : return madvise_populate(vma, prev, start, end, behavior);
1027 : case MADV_NORMAL:
1028 0 : new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
1029 0 : break;
1030 : case MADV_SEQUENTIAL:
1031 0 : new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
1032 0 : break;
1033 : case MADV_RANDOM:
1034 0 : new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
1035 0 : break;
1036 : case MADV_DONTFORK:
1037 0 : new_flags |= VM_DONTCOPY;
1038 0 : break;
1039 : case MADV_DOFORK:
1040 0 : if (vma->vm_flags & VM_IO)
1041 : return -EINVAL;
1042 0 : new_flags &= ~VM_DONTCOPY;
1043 0 : break;
1044 : case MADV_WIPEONFORK:
1045 : /* MADV_WIPEONFORK is only supported on anonymous memory. */
1046 0 : if (vma->vm_file || vma->vm_flags & VM_SHARED)
1047 : return -EINVAL;
1048 0 : new_flags |= VM_WIPEONFORK;
1049 0 : break;
1050 : case MADV_KEEPONFORK:
1051 0 : new_flags &= ~VM_WIPEONFORK;
1052 0 : break;
1053 : case MADV_DONTDUMP:
1054 0 : new_flags |= VM_DONTDUMP;
1055 0 : break;
1056 : case MADV_DODUMP:
1057 0 : if (!is_vm_hugetlb_page(vma) && new_flags & VM_SPECIAL)
1058 : return -EINVAL;
1059 0 : new_flags &= ~VM_DONTDUMP;
1060 0 : break;
1061 : case MADV_MERGEABLE:
1062 : case MADV_UNMERGEABLE:
1063 : error = ksm_madvise(vma, start, end, behavior, &new_flags);
1064 : if (error)
1065 : goto out;
1066 : break;
1067 : case MADV_HUGEPAGE:
1068 : case MADV_NOHUGEPAGE:
1069 : error = hugepage_madvise(vma, &new_flags, behavior);
1070 : if (error)
1071 : goto out;
1072 : break;
1073 : case MADV_COLLAPSE:
1074 : return madvise_collapse(vma, prev, start, end);
1075 : }
1076 :
1077 0 : anon_name = anon_vma_name(vma);
1078 : anon_vma_name_get(anon_name);
1079 0 : error = madvise_update_vma(vma, prev, start, end, new_flags,
1080 : anon_name);
1081 : anon_vma_name_put(anon_name);
1082 :
1083 : out:
1084 : /*
1085 : * madvise() returns EAGAIN if kernel resources, such as
1086 : * slab, are temporarily unavailable.
1087 : */
1088 0 : if (error == -ENOMEM)
1089 0 : error = -EAGAIN;
1090 : return error;
1091 : }
1092 :
1093 : #ifdef CONFIG_MEMORY_FAILURE
1094 : /*
1095 : * Error injection support for memory error handling.
1096 : */
1097 : static int madvise_inject_error(int behavior,
1098 : unsigned long start, unsigned long end)
1099 : {
1100 : unsigned long size;
1101 :
1102 : if (!capable(CAP_SYS_ADMIN))
1103 : return -EPERM;
1104 :
1105 :
1106 : for (; start < end; start += size) {
1107 : unsigned long pfn;
1108 : struct page *page;
1109 : int ret;
1110 :
1111 : ret = get_user_pages_fast(start, 1, 0, &page);
1112 : if (ret != 1)
1113 : return ret;
1114 : pfn = page_to_pfn(page);
1115 :
1116 : /*
1117 : * When soft offlining hugepages, after migrating the page
1118 : * we dissolve it, therefore in the second loop "page" will
1119 : * no longer be a compound page.
1120 : */
1121 : size = page_size(compound_head(page));
1122 :
1123 : if (behavior == MADV_SOFT_OFFLINE) {
1124 : pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
1125 : pfn, start);
1126 : ret = soft_offline_page(pfn, MF_COUNT_INCREASED);
1127 : } else {
1128 : pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n",
1129 : pfn, start);
1130 : ret = memory_failure(pfn, MF_COUNT_INCREASED | MF_SW_SIMULATED);
1131 : if (ret == -EOPNOTSUPP)
1132 : ret = 0;
1133 : }
1134 :
1135 : if (ret)
1136 : return ret;
1137 : }
1138 :
1139 : return 0;
1140 : }
1141 : #endif
1142 :
1143 : static bool
1144 : madvise_behavior_valid(int behavior)
1145 : {
1146 : switch (behavior) {
1147 : case MADV_DOFORK:
1148 : case MADV_DONTFORK:
1149 : case MADV_NORMAL:
1150 : case MADV_SEQUENTIAL:
1151 : case MADV_RANDOM:
1152 : case MADV_REMOVE:
1153 : case MADV_WILLNEED:
1154 : case MADV_DONTNEED:
1155 : case MADV_DONTNEED_LOCKED:
1156 : case MADV_FREE:
1157 : case MADV_COLD:
1158 : case MADV_PAGEOUT:
1159 : case MADV_POPULATE_READ:
1160 : case MADV_POPULATE_WRITE:
1161 : #ifdef CONFIG_KSM
1162 : case MADV_MERGEABLE:
1163 : case MADV_UNMERGEABLE:
1164 : #endif
1165 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1166 : case MADV_HUGEPAGE:
1167 : case MADV_NOHUGEPAGE:
1168 : case MADV_COLLAPSE:
1169 : #endif
1170 : case MADV_DONTDUMP:
1171 : case MADV_DODUMP:
1172 : case MADV_WIPEONFORK:
1173 : case MADV_KEEPONFORK:
1174 : #ifdef CONFIG_MEMORY_FAILURE
1175 : case MADV_SOFT_OFFLINE:
1176 : case MADV_HWPOISON:
1177 : #endif
1178 : return true;
1179 :
1180 : default:
1181 : return false;
1182 : }
1183 : }
1184 :
1185 : static bool process_madvise_behavior_valid(int behavior)
1186 : {
1187 : switch (behavior) {
1188 : case MADV_COLD:
1189 : case MADV_PAGEOUT:
1190 : case MADV_WILLNEED:
1191 : case MADV_COLLAPSE:
1192 : return true;
1193 : default:
1194 : return false;
1195 : }
1196 : }
1197 :
1198 : /*
1199 : * Walk the vmas in range [start,end), and call the visit function on each one.
1200 : * The visit function will get start and end parameters that cover the overlap
1201 : * between the current vma and the original range. Any unmapped regions in the
1202 : * original range will result in this function returning -ENOMEM while still
1203 : * calling the visit function on all of the existing vmas in the range.
1204 : * Must be called with the mmap_lock held for reading or writing.
1205 : */
1206 : static
1207 0 : int madvise_walk_vmas(struct mm_struct *mm, unsigned long start,
1208 : unsigned long end, unsigned long arg,
1209 : int (*visit)(struct vm_area_struct *vma,
1210 : struct vm_area_struct **prev, unsigned long start,
1211 : unsigned long end, unsigned long arg))
1212 : {
1213 : struct vm_area_struct *vma;
1214 : struct vm_area_struct *prev;
1215 : unsigned long tmp;
1216 0 : int unmapped_error = 0;
1217 :
1218 : /*
1219 : * If the interval [start,end) covers some unmapped address
1220 : * ranges, just ignore them, but return -ENOMEM at the end.
1221 : * - different from the way of handling in mlock etc.
1222 : */
1223 0 : vma = find_vma_prev(mm, start, &prev);
1224 0 : if (vma && start > vma->vm_start)
1225 0 : prev = vma;
1226 :
1227 : for (;;) {
1228 : int error;
1229 :
1230 : /* Still start < end. */
1231 0 : if (!vma)
1232 : return -ENOMEM;
1233 :
1234 : /* Here start < (end|vma->vm_end). */
1235 0 : if (start < vma->vm_start) {
1236 0 : unmapped_error = -ENOMEM;
1237 0 : start = vma->vm_start;
1238 0 : if (start >= end)
1239 : break;
1240 : }
1241 :
1242 : /* Here vma->vm_start <= start < (end|vma->vm_end) */
1243 0 : tmp = vma->vm_end;
1244 0 : if (end < tmp)
1245 0 : tmp = end;
1246 :
1247 : /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
1248 0 : error = visit(vma, &prev, start, tmp, arg);
1249 0 : if (error)
1250 : return error;
1251 0 : start = tmp;
1252 0 : if (prev && start < prev->vm_end)
1253 0 : start = prev->vm_end;
1254 0 : if (start >= end)
1255 : break;
1256 0 : if (prev)
1257 0 : vma = find_vma(mm, prev->vm_end);
1258 : else /* madvise_remove dropped mmap_lock */
1259 0 : vma = find_vma(mm, start);
1260 : }
1261 :
1262 : return unmapped_error;
1263 : }
1264 :
1265 : #ifdef CONFIG_ANON_VMA_NAME
1266 : static int madvise_vma_anon_name(struct vm_area_struct *vma,
1267 : struct vm_area_struct **prev,
1268 : unsigned long start, unsigned long end,
1269 : unsigned long anon_name)
1270 : {
1271 : int error;
1272 :
1273 : /* Only anonymous mappings can be named */
1274 : if (vma->vm_file && !vma_is_anon_shmem(vma))
1275 : return -EBADF;
1276 :
1277 : error = madvise_update_vma(vma, prev, start, end, vma->vm_flags,
1278 : (struct anon_vma_name *)anon_name);
1279 :
1280 : /*
1281 : * madvise() returns EAGAIN if kernel resources, such as
1282 : * slab, are temporarily unavailable.
1283 : */
1284 : if (error == -ENOMEM)
1285 : error = -EAGAIN;
1286 : return error;
1287 : }
1288 :
1289 : int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
1290 : unsigned long len_in, struct anon_vma_name *anon_name)
1291 : {
1292 : unsigned long end;
1293 : unsigned long len;
1294 :
1295 : if (start & ~PAGE_MASK)
1296 : return -EINVAL;
1297 : len = (len_in + ~PAGE_MASK) & PAGE_MASK;
1298 :
1299 : /* Check to see whether len was rounded up from small -ve to zero */
1300 : if (len_in && !len)
1301 : return -EINVAL;
1302 :
1303 : end = start + len;
1304 : if (end < start)
1305 : return -EINVAL;
1306 :
1307 : if (end == start)
1308 : return 0;
1309 :
1310 : return madvise_walk_vmas(mm, start, end, (unsigned long)anon_name,
1311 : madvise_vma_anon_name);
1312 : }
1313 : #endif /* CONFIG_ANON_VMA_NAME */
1314 : /*
1315 : * The madvise(2) system call.
1316 : *
1317 : * Applications can use madvise() to advise the kernel how it should
1318 : * handle paging I/O in this VM area. The idea is to help the kernel
1319 : * use appropriate read-ahead and caching techniques. The information
1320 : * provided is advisory only, and can be safely disregarded by the
1321 : * kernel without affecting the correct operation of the application.
1322 : *
1323 : * behavior values:
1324 : * MADV_NORMAL - the default behavior is to read clusters. This
1325 : * results in some read-ahead and read-behind.
1326 : * MADV_RANDOM - the system should read the minimum amount of data
1327 : * on any access, since it is unlikely that the appli-
1328 : * cation will need more than what it asks for.
1329 : * MADV_SEQUENTIAL - pages in the given range will probably be accessed
1330 : * once, so they can be aggressively read ahead, and
1331 : * can be freed soon after they are accessed.
1332 : * MADV_WILLNEED - the application is notifying the system to read
1333 : * some pages ahead.
1334 : * MADV_DONTNEED - the application is finished with the given range,
1335 : * so the kernel can free resources associated with it.
1336 : * MADV_FREE - the application marks pages in the given range as lazy free,
1337 : * where actual purges are postponed until memory pressure happens.
1338 : * MADV_REMOVE - the application wants to free up the given range of
1339 : * pages and associated backing store.
1340 : * MADV_DONTFORK - omit this area from child's address space when forking:
1341 : * typically, to avoid COWing pages pinned by get_user_pages().
1342 : * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
1343 : * MADV_WIPEONFORK - present the child process with zero-filled memory in this
1344 : * range after a fork.
1345 : * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK
1346 : * MADV_HWPOISON - trigger memory error handler as if the given memory range
1347 : * were corrupted by unrecoverable hardware memory failure.
1348 : * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
1349 : * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
1350 : * this area with pages of identical content from other such areas.
1351 : * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
1352 : * MADV_HUGEPAGE - the application wants to back the given range by transparent
1353 : * huge pages in the future. Existing pages might be coalesced and
1354 : * new pages might be allocated as THP.
1355 : * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
1356 : * transparent huge pages so the existing pages will not be
1357 : * coalesced into THP and new pages will not be allocated as THP.
1358 : * MADV_COLLAPSE - synchronously coalesce pages into new THP.
1359 : * MADV_DONTDUMP - the application wants to prevent pages in the given range
1360 : * from being included in its core dump.
1361 : * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
1362 : * MADV_COLD - the application is not expected to use this memory soon,
1363 : * deactivate pages in this range so that they can be reclaimed
1364 : * easily if memory pressure happens.
1365 : * MADV_PAGEOUT - the application is not expected to use this memory soon,
1366 : * page out the pages in this range immediately.
1367 : * MADV_POPULATE_READ - populate (prefault) page tables readable by
1368 : * triggering read faults if required
1369 : * MADV_POPULATE_WRITE - populate (prefault) page tables writable by
1370 : * triggering write faults if required
1371 : *
1372 : * return values:
1373 : * zero - success
1374 : * -EINVAL - start + len < 0, start is not page-aligned,
1375 : * "behavior" is not a valid value, or application
1376 : * is attempting to release locked or shared pages,
1377 : * or the specified address range includes file, Huge TLB,
1378 : * MAP_SHARED or VMPFNMAP range.
1379 : * -ENOMEM - addresses in the specified range are not currently
1380 : * mapped, or are outside the AS of the process.
1381 : * -EIO - an I/O error occurred while paging in data.
1382 : * -EBADF - map exists, but area maps something that isn't a file.
1383 : * -EAGAIN - a kernel resource was temporarily unavailable.
1384 : */
1385 0 : int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior)
1386 : {
1387 : unsigned long end;
1388 : int error;
1389 : int write;
1390 : size_t len;
1391 : struct blk_plug plug;
1392 :
1393 0 : if (!madvise_behavior_valid(behavior))
1394 : return -EINVAL;
1395 :
1396 0 : if (!PAGE_ALIGNED(start))
1397 : return -EINVAL;
1398 0 : len = PAGE_ALIGN(len_in);
1399 :
1400 : /* Check to see whether len was rounded up from small -ve to zero */
1401 0 : if (len_in && !len)
1402 : return -EINVAL;
1403 :
1404 0 : end = start + len;
1405 0 : if (end < start)
1406 : return -EINVAL;
1407 :
1408 0 : if (end == start)
1409 : return 0;
1410 :
1411 : #ifdef CONFIG_MEMORY_FAILURE
1412 : if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
1413 : return madvise_inject_error(behavior, start, start + len_in);
1414 : #endif
1415 :
1416 0 : write = madvise_need_mmap_write(behavior);
1417 0 : if (write) {
1418 0 : if (mmap_write_lock_killable(mm))
1419 : return -EINTR;
1420 : } else {
1421 : mmap_read_lock(mm);
1422 : }
1423 :
1424 0 : start = untagged_addr_remote(mm, start);
1425 0 : end = start + len;
1426 :
1427 0 : blk_start_plug(&plug);
1428 0 : error = madvise_walk_vmas(mm, start, end, behavior,
1429 : madvise_vma_behavior);
1430 0 : blk_finish_plug(&plug);
1431 0 : if (write)
1432 : mmap_write_unlock(mm);
1433 : else
1434 : mmap_read_unlock(mm);
1435 :
1436 : return error;
1437 : }
1438 :
1439 0 : SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
1440 : {
1441 0 : return do_madvise(current->mm, start, len_in, behavior);
1442 : }
1443 :
1444 0 : SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
1445 : size_t, vlen, int, behavior, unsigned int, flags)
1446 : {
1447 : ssize_t ret;
1448 : struct iovec iovstack[UIO_FASTIOV];
1449 0 : struct iovec *iov = iovstack;
1450 : struct iov_iter iter;
1451 : struct task_struct *task;
1452 : struct mm_struct *mm;
1453 : size_t total_len;
1454 : unsigned int f_flags;
1455 :
1456 0 : if (flags != 0) {
1457 : ret = -EINVAL;
1458 : goto out;
1459 : }
1460 :
1461 0 : ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
1462 0 : if (ret < 0)
1463 : goto out;
1464 :
1465 0 : task = pidfd_get_task(pidfd, &f_flags);
1466 0 : if (IS_ERR(task)) {
1467 0 : ret = PTR_ERR(task);
1468 0 : goto free_iov;
1469 : }
1470 :
1471 0 : if (!process_madvise_behavior_valid(behavior)) {
1472 : ret = -EINVAL;
1473 : goto release_task;
1474 : }
1475 :
1476 : /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */
1477 0 : mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
1478 0 : if (IS_ERR_OR_NULL(mm)) {
1479 0 : ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
1480 : goto release_task;
1481 : }
1482 :
1483 : /*
1484 : * Require CAP_SYS_NICE for influencing process performance. Note that
1485 : * only non-destructive hints are currently supported.
1486 : */
1487 0 : if (!capable(CAP_SYS_NICE)) {
1488 : ret = -EPERM;
1489 : goto release_mm;
1490 : }
1491 :
1492 0 : total_len = iov_iter_count(&iter);
1493 :
1494 0 : while (iov_iter_count(&iter)) {
1495 0 : ret = do_madvise(mm, (unsigned long)iter_iov_addr(&iter),
1496 0 : iter_iov_len(&iter), behavior);
1497 0 : if (ret < 0)
1498 : break;
1499 0 : iov_iter_advance(&iter, iter_iov_len(&iter));
1500 : }
1501 :
1502 0 : ret = (total_len - iov_iter_count(&iter)) ? : ret;
1503 :
1504 : release_mm:
1505 0 : mmput(mm);
1506 : release_task:
1507 0 : put_task_struct(task);
1508 : free_iov:
1509 0 : kfree(iov);
1510 : out:
1511 0 : return ret;
1512 : }
|
