LCOV - code coverage report
Current view: top level - mm - mprotect.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 0 229 0.0 %
Date: 2023-08-24 13:40:31 Functions: 0 9 0.0 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  *  mm/mprotect.c
       4             :  *
       5             :  *  (C) Copyright 1994 Linus Torvalds
       6             :  *  (C) Copyright 2002 Christoph Hellwig
       7             :  *
       8             :  *  Address space accounting code       <alan@lxorguk.ukuu.org.uk>
       9             :  *  (C) Copyright 2002 Red Hat Inc, All Rights Reserved
      10             :  */
      11             : 
      12             : #include <linux/pagewalk.h>
      13             : #include <linux/hugetlb.h>
      14             : #include <linux/shm.h>
      15             : #include <linux/mman.h>
      16             : #include <linux/fs.h>
      17             : #include <linux/highmem.h>
      18             : #include <linux/security.h>
      19             : #include <linux/mempolicy.h>
      20             : #include <linux/personality.h>
      21             : #include <linux/syscalls.h>
      22             : #include <linux/swap.h>
      23             : #include <linux/swapops.h>
      24             : #include <linux/mmu_notifier.h>
      25             : #include <linux/migrate.h>
      26             : #include <linux/perf_event.h>
      27             : #include <linux/pkeys.h>
      28             : #include <linux/ksm.h>
      29             : #include <linux/uaccess.h>
      30             : #include <linux/mm_inline.h>
      31             : #include <linux/pgtable.h>
      32             : #include <linux/sched/sysctl.h>
      33             : #include <linux/userfaultfd_k.h>
      34             : #include <linux/memory-tiers.h>
      35             : #include <asm/cacheflush.h>
      36             : #include <asm/mmu_context.h>
      37             : #include <asm/tlbflush.h>
      38             : #include <asm/tlb.h>
      39             : 
      40             : #include "internal.h"
      41             : 
      42           0 : bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
      43             :                              pte_t pte)
      44             : {
      45             :         struct page *page;
      46             : 
      47           0 :         if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
      48             :                 return false;
      49             : 
      50             :         /* Don't touch entries that are not even readable. */
      51           0 :         if (pte_protnone(pte))
      52             :                 return false;
      53             : 
      54             :         /* Do we need write faults for softdirty tracking? */
      55           0 :         if (vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte))
      56             :                 return false;
      57             : 
      58             :         /* Do we need write faults for uffd-wp tracking? */
      59           0 :         if (userfaultfd_pte_wp(vma, pte))
      60             :                 return false;
      61             : 
      62           0 :         if (!(vma->vm_flags & VM_SHARED)) {
      63             :                 /*
      64             :                  * Writable MAP_PRIVATE mapping: We can only special-case on
      65             :                  * exclusive anonymous pages, because we know that our
      66             :                  * write-fault handler similarly would map them writable without
      67             :                  * any additional checks while holding the PT lock.
      68             :                  */
      69           0 :                 page = vm_normal_page(vma, addr, pte);
      70           0 :                 return page && PageAnon(page) && PageAnonExclusive(page);
      71             :         }
      72             : 
      73             :         /*
      74             :          * Writable MAP_SHARED mapping: "clean" might indicate that the FS still
      75             :          * needs a real write-fault for writenotify
      76             :          * (see vma_wants_writenotify()). If "dirty", the assumption is that the
      77             :          * FS was already notified and we can simply mark the PTE writable
      78             :          * just like the write-fault handler would do.
      79             :          */
      80           0 :         return pte_dirty(pte);
      81             : }
      82             : 
      83           0 : static long change_pte_range(struct mmu_gather *tlb,
      84             :                 struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
      85             :                 unsigned long end, pgprot_t newprot, unsigned long cp_flags)
      86             : {
      87             :         pte_t *pte, oldpte;
      88             :         spinlock_t *ptl;
      89           0 :         long pages = 0;
      90           0 :         int target_node = NUMA_NO_NODE;
      91           0 :         bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
      92           0 :         bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
      93           0 :         bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
      94             : 
      95           0 :         tlb_change_page_size(tlb, PAGE_SIZE);
      96           0 :         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
      97           0 :         if (!pte)
      98             :                 return -EAGAIN;
      99             : 
     100             :         /* Get target node for single threaded private VMAs */
     101           0 :         if (prot_numa && !(vma->vm_flags & VM_SHARED) &&
     102           0 :             atomic_read(&vma->vm_mm->mm_users) == 1)
     103           0 :                 target_node = numa_node_id();
     104             : 
     105           0 :         flush_tlb_batched_pending(vma->vm_mm);
     106             :         arch_enter_lazy_mmu_mode();
     107             :         do {
     108           0 :                 oldpte = ptep_get(pte);
     109           0 :                 if (pte_present(oldpte)) {
     110             :                         pte_t ptent;
     111             : 
     112             :                         /*
     113             :                          * Avoid trapping faults against the zero or KSM
     114             :                          * pages. See similar comment in change_huge_pmd.
     115             :                          */
     116           0 :                         if (prot_numa) {
     117             :                                 struct page *page;
     118             :                                 int nid;
     119             :                                 bool toptier;
     120             : 
     121             :                                 /* Avoid TLB flush if possible */
     122           0 :                                 if (pte_protnone(oldpte))
     123             :                                         continue;
     124             : 
     125           0 :                                 page = vm_normal_page(vma, addr, oldpte);
     126           0 :                                 if (!page || is_zone_device_page(page) || PageKsm(page))
     127           0 :                                         continue;
     128             : 
     129             :                                 /* Also skip shared copy-on-write pages */
     130           0 :                                 if (is_cow_mapping(vma->vm_flags) &&
     131           0 :                                     page_count(page) != 1)
     132           0 :                                         continue;
     133             : 
     134             :                                 /*
     135             :                                  * While migration can move some dirty pages,
     136             :                                  * it cannot move them all from MIGRATE_ASYNC
     137             :                                  * context.
     138             :                                  */
     139           0 :                                 if (page_is_file_lru(page) && PageDirty(page))
     140           0 :                                         continue;
     141             : 
     142             :                                 /*
     143             :                                  * Don't mess with PTEs if page is already on the node
     144             :                                  * a single-threaded process is running on.
     145             :                                  */
     146           0 :                                 nid = page_to_nid(page);
     147           0 :                                 if (target_node == nid)
     148           0 :                                         continue;
     149           0 :                                 toptier = node_is_toptier(nid);
     150             : 
     151             :                                 /*
     152             :                                  * Skip scanning top tier node if normal numa
     153             :                                  * balancing is disabled
     154             :                                  */
     155             :                                 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
     156             :                                     toptier)
     157           0 :                                         continue;
     158             :                                 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
     159             :                                     !toptier)
     160             :                                         xchg_page_access_time(page,
     161             :                                                 jiffies_to_msecs(jiffies));
     162             :                         }
     163             : 
     164           0 :                         oldpte = ptep_modify_prot_start(vma, addr, pte);
     165           0 :                         ptent = pte_modify(oldpte, newprot);
     166             : 
     167             :                         if (uffd_wp)
     168             :                                 ptent = pte_mkuffd_wp(ptent);
     169             :                         else if (uffd_wp_resolve)
     170             :                                 ptent = pte_clear_uffd_wp(ptent);
     171             : 
     172             :                         /*
     173             :                          * In some writable, shared mappings, we might want
     174             :                          * to catch actual write access -- see
     175             :                          * vma_wants_writenotify().
     176             :                          *
     177             :                          * In all writable, private mappings, we have to
     178             :                          * properly handle COW.
     179             :                          *
     180             :                          * In both cases, we can sometimes still change PTEs
     181             :                          * writable and avoid the write-fault handler, for
     182             :                          * example, if a PTE is already dirty and no other
     183             :                          * COW or special handling is required.
     184             :                          */
     185           0 :                         if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
     186           0 :                             !pte_write(ptent) &&
     187           0 :                             can_change_pte_writable(vma, addr, ptent))
     188             :                                 ptent = pte_mkwrite(ptent);
     189             : 
     190           0 :                         ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
     191           0 :                         if (pte_needs_flush(oldpte, ptent))
     192             :                                 tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
     193           0 :                         pages++;
     194           0 :                 } else if (is_swap_pte(oldpte)) {
     195           0 :                         swp_entry_t entry = pte_to_swp_entry(oldpte);
     196             :                         pte_t newpte;
     197             : 
     198           0 :                         if (is_writable_migration_entry(entry)) {
     199           0 :                                 struct page *page = pfn_swap_entry_to_page(entry);
     200             : 
     201             :                                 /*
     202             :                                  * A protection check is difficult so
     203             :                                  * just be safe and disable write
     204             :                                  */
     205           0 :                                 if (PageAnon(page))
     206           0 :                                         entry = make_readable_exclusive_migration_entry(
     207             :                                                              swp_offset(entry));
     208             :                                 else
     209           0 :                                         entry = make_readable_migration_entry(swp_offset(entry));
     210           0 :                                 newpte = swp_entry_to_pte(entry);
     211           0 :                                 if (pte_swp_soft_dirty(oldpte))
     212             :                                         newpte = pte_swp_mksoft_dirty(newpte);
     213           0 :                         } else if (is_writable_device_private_entry(entry)) {
     214             :                                 /*
     215             :                                  * We do not preserve soft-dirtiness. See
     216             :                                  * copy_one_pte() for explanation.
     217             :                                  */
     218             :                                 entry = make_readable_device_private_entry(
     219             :                                                         swp_offset(entry));
     220             :                                 newpte = swp_entry_to_pte(entry);
     221             :                                 if (pte_swp_uffd_wp(oldpte))
     222             :                                         newpte = pte_swp_mkuffd_wp(newpte);
     223           0 :                         } else if (is_writable_device_exclusive_entry(entry)) {
     224             :                                 entry = make_readable_device_exclusive_entry(
     225             :                                                         swp_offset(entry));
     226             :                                 newpte = swp_entry_to_pte(entry);
     227             :                                 if (pte_swp_soft_dirty(oldpte))
     228             :                                         newpte = pte_swp_mksoft_dirty(newpte);
     229             :                                 if (pte_swp_uffd_wp(oldpte))
     230             :                                         newpte = pte_swp_mkuffd_wp(newpte);
     231           0 :                         } else if (is_pte_marker_entry(entry)) {
     232             :                                 /*
     233             :                                  * Ignore swapin errors unconditionally,
     234             :                                  * because any access should sigbus anyway.
     235             :                                  */
     236           0 :                                 if (is_swapin_error_entry(entry))
     237           0 :                                         continue;
     238             :                                 /*
     239             :                                  * If this is uffd-wp pte marker and we'd like
     240             :                                  * to unprotect it, drop it; the next page
     241             :                                  * fault will trigger without uffd trapping.
     242             :                                  */
     243           0 :                                 if (uffd_wp_resolve) {
     244           0 :                                         pte_clear(vma->vm_mm, addr, pte);
     245           0 :                                         pages++;
     246             :                                 }
     247           0 :                                 continue;
     248             :                         } else {
     249             :                                 newpte = oldpte;
     250             :                         }
     251             : 
     252             :                         if (uffd_wp)
     253             :                                 newpte = pte_swp_mkuffd_wp(newpte);
     254             :                         else if (uffd_wp_resolve)
     255             :                                 newpte = pte_swp_clear_uffd_wp(newpte);
     256             : 
     257           0 :                         if (!pte_same(oldpte, newpte)) {
     258           0 :                                 set_pte_at(vma->vm_mm, addr, pte, newpte);
     259           0 :                                 pages++;
     260             :                         }
     261             :                 } else {
     262             :                         /* It must be an none page, or what else?.. */
     263           0 :                         WARN_ON_ONCE(!pte_none(oldpte));
     264             : 
     265             :                         /*
     266             :                          * Nobody plays with any none ptes besides
     267             :                          * userfaultfd when applying the protections.
     268             :                          */
     269           0 :                         if (likely(!uffd_wp))
     270           0 :                                 continue;
     271             : 
     272             :                         if (userfaultfd_wp_use_markers(vma)) {
     273             :                                 /*
     274             :                                  * For file-backed mem, we need to be able to
     275             :                                  * wr-protect a none pte, because even if the
     276             :                                  * pte is none, the page/swap cache could
     277             :                                  * exist.  Doing that by install a marker.
     278             :                                  */
     279             :                                 set_pte_at(vma->vm_mm, addr, pte,
     280             :                                            make_pte_marker(PTE_MARKER_UFFD_WP));
     281             :                                 pages++;
     282             :                         }
     283             :                 }
     284           0 :         } while (pte++, addr += PAGE_SIZE, addr != end);
     285             :         arch_leave_lazy_mmu_mode();
     286           0 :         pte_unmap_unlock(pte - 1, ptl);
     287             : 
     288           0 :         return pages;
     289             : }
     290             : 
     291             : /*
     292             :  * Return true if we want to split THPs into PTE mappings in change
     293             :  * protection procedure, false otherwise.
     294             :  */
     295             : static inline bool
     296             : pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags)
     297             : {
     298             :         /*
     299             :          * pte markers only resides in pte level, if we need pte markers,
     300             :          * we need to split.  We cannot wr-protect shmem thp because file
     301             :          * thp is handled differently when split by erasing the pmd so far.
     302             :          */
     303             :         return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma);
     304             : }
     305             : 
     306             : /*
     307             :  * Return true if we want to populate pgtables in change protection
     308             :  * procedure, false otherwise
     309             :  */
     310             : static inline bool
     311             : pgtable_populate_needed(struct vm_area_struct *vma, unsigned long cp_flags)
     312             : {
     313             :         /* If not within ioctl(UFFDIO_WRITEPROTECT), then don't bother */
     314             :         if (!(cp_flags & MM_CP_UFFD_WP))
     315             :                 return false;
     316             : 
     317             :         /* Populate if the userfaultfd mode requires pte markers */
     318             :         return userfaultfd_wp_use_markers(vma);
     319             : }
     320             : 
     321             : /*
     322             :  * Populate the pgtable underneath for whatever reason if requested.
     323             :  * When {pte|pmd|...}_alloc() failed we treat it the same way as pgtable
     324             :  * allocation failures during page faults by kicking OOM and returning
     325             :  * error.
     326             :  */
     327             : #define  change_pmd_prepare(vma, pmd, cp_flags)                         \
     328             :         ({                                                              \
     329             :                 long err = 0;                                           \
     330             :                 if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
     331             :                         if (pte_alloc(vma->vm_mm, pmd))                      \
     332             :                                 err = -ENOMEM;                          \
     333             :                 }                                                       \
     334             :                 err;                                                    \
     335             :         })
     336             : 
     337             : /*
     338             :  * This is the general pud/p4d/pgd version of change_pmd_prepare(). We need to
     339             :  * have separate change_pmd_prepare() because pte_alloc() returns 0 on success,
     340             :  * while {pmd|pud|p4d}_alloc() returns the valid pointer on success.
     341             :  */
     342             : #define  change_prepare(vma, high, low, addr, cp_flags)                 \
     343             :           ({                                                            \
     344             :                 long err = 0;                                           \
     345             :                 if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
     346             :                         low##_t *p = low##_alloc(vma->vm_mm, high, addr); \
     347             :                         if (p == NULL)                                  \
     348             :                                 err = -ENOMEM;                          \
     349             :                 }                                                       \
     350             :                 err;                                                    \
     351             :         })
     352             : 
     353           0 : static inline long change_pmd_range(struct mmu_gather *tlb,
     354             :                 struct vm_area_struct *vma, pud_t *pud, unsigned long addr,
     355             :                 unsigned long end, pgprot_t newprot, unsigned long cp_flags)
     356             : {
     357             :         pmd_t *pmd;
     358             :         unsigned long next;
     359           0 :         long pages = 0;
     360           0 :         unsigned long nr_huge_updates = 0;
     361             :         struct mmu_notifier_range range;
     362             : 
     363           0 :         range.start = 0;
     364             : 
     365           0 :         pmd = pmd_offset(pud, addr);
     366             :         do {
     367             :                 long ret;
     368             :                 pmd_t _pmd;
     369             : again:
     370           0 :                 next = pmd_addr_end(addr, end);
     371             : 
     372           0 :                 ret = change_pmd_prepare(vma, pmd, cp_flags);
     373             :                 if (ret) {
     374             :                         pages = ret;
     375             :                         break;
     376             :                 }
     377             : 
     378           0 :                 if (pmd_none(*pmd))
     379             :                         goto next;
     380             : 
     381             :                 /* invoke the mmu notifier if the pmd is populated */
     382             :                 if (!range.start) {
     383             :                         mmu_notifier_range_init(&range,
     384             :                                 MMU_NOTIFY_PROTECTION_VMA, 0,
     385             :                                 vma->vm_mm, addr, end);
     386             :                         mmu_notifier_invalidate_range_start(&range);
     387             :                 }
     388             : 
     389           0 :                 _pmd = pmdp_get_lockless(pmd);
     390           0 :                 if (is_swap_pmd(_pmd) || pmd_trans_huge(_pmd) || pmd_devmap(_pmd)) {
     391             :                         if ((next - addr != HPAGE_PMD_SIZE) ||
     392             :                             pgtable_split_needed(vma, cp_flags)) {
     393             :                                 __split_huge_pmd(vma, pmd, addr, false, NULL);
     394             :                                 /*
     395             :                                  * For file-backed, the pmd could have been
     396             :                                  * cleared; make sure pmd populated if
     397             :                                  * necessary, then fall-through to pte level.
     398             :                                  */
     399             :                                 ret = change_pmd_prepare(vma, pmd, cp_flags);
     400             :                                 if (ret) {
     401             :                                         pages = ret;
     402             :                                         break;
     403             :                                 }
     404             :                         } else {
     405             :                                 ret = change_huge_pmd(tlb, vma, pmd,
     406             :                                                 addr, newprot, cp_flags);
     407             :                                 if (ret) {
     408             :                                         if (ret == HPAGE_PMD_NR) {
     409             :                                                 pages += HPAGE_PMD_NR;
     410             :                                                 nr_huge_updates++;
     411             :                                         }
     412             : 
     413             :                                         /* huge pmd was handled */
     414             :                                         goto next;
     415             :                                 }
     416             :                         }
     417             :                         /* fall through, the trans huge pmd just split */
     418             :                 }
     419             : 
     420           0 :                 ret = change_pte_range(tlb, vma, pmd, addr, next, newprot,
     421             :                                        cp_flags);
     422           0 :                 if (ret < 0)
     423             :                         goto again;
     424           0 :                 pages += ret;
     425             : next:
     426           0 :                 cond_resched();
     427           0 :         } while (pmd++, addr = next, addr != end);
     428             : 
     429             :         if (range.start)
     430             :                 mmu_notifier_invalidate_range_end(&range);
     431             : 
     432             :         if (nr_huge_updates)
     433             :                 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
     434           0 :         return pages;
     435             : }
     436             : 
     437           0 : static inline long change_pud_range(struct mmu_gather *tlb,
     438             :                 struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr,
     439             :                 unsigned long end, pgprot_t newprot, unsigned long cp_flags)
     440             : {
     441             :         pud_t *pud;
     442             :         unsigned long next;
     443           0 :         long pages = 0, ret;
     444             : 
     445           0 :         pud = pud_offset(p4d, addr);
     446             :         do {
     447           0 :                 next = pud_addr_end(addr, end);
     448           0 :                 ret = change_prepare(vma, pud, pmd, addr, cp_flags);
     449             :                 if (ret)
     450             :                         return ret;
     451           0 :                 if (pud_none_or_clear_bad(pud))
     452           0 :                         continue;
     453           0 :                 pages += change_pmd_range(tlb, vma, pud, addr, next, newprot,
     454             :                                           cp_flags);
     455           0 :         } while (pud++, addr = next, addr != end);
     456             : 
     457             :         return pages;
     458             : }
     459             : 
     460             : static inline long change_p4d_range(struct mmu_gather *tlb,
     461             :                 struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr,
     462             :                 unsigned long end, pgprot_t newprot, unsigned long cp_flags)
     463             : {
     464             :         p4d_t *p4d;
     465             :         unsigned long next;
     466           0 :         long pages = 0, ret;
     467             : 
     468           0 :         p4d = p4d_offset(pgd, addr);
     469             :         do {
     470           0 :                 next = p4d_addr_end(addr, end);
     471           0 :                 ret = change_prepare(vma, p4d, pud, addr, cp_flags);
     472             :                 if (ret)
     473             :                         return ret;
     474           0 :                 if (p4d_none_or_clear_bad(p4d))
     475             :                         continue;
     476           0 :                 pages += change_pud_range(tlb, vma, p4d, addr, next, newprot,
     477             :                                           cp_flags);
     478           0 :         } while (p4d++, addr = next, addr != end);
     479             : 
     480             :         return pages;
     481             : }
     482             : 
     483           0 : static long change_protection_range(struct mmu_gather *tlb,
     484             :                 struct vm_area_struct *vma, unsigned long addr,
     485             :                 unsigned long end, pgprot_t newprot, unsigned long cp_flags)
     486             : {
     487           0 :         struct mm_struct *mm = vma->vm_mm;
     488             :         pgd_t *pgd;
     489             :         unsigned long next;
     490           0 :         long pages = 0, ret;
     491             : 
     492           0 :         BUG_ON(addr >= end);
     493           0 :         pgd = pgd_offset(mm, addr);
     494             :         tlb_start_vma(tlb, vma);
     495             :         do {
     496           0 :                 next = pgd_addr_end(addr, end);
     497           0 :                 ret = change_prepare(vma, pgd, p4d, addr, cp_flags);
     498             :                 if (ret) {
     499             :                         pages = ret;
     500             :                         break;
     501             :                 }
     502           0 :                 if (pgd_none_or_clear_bad(pgd))
     503             :                         continue;
     504           0 :                 pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot,
     505             :                                           cp_flags);
     506           0 :         } while (pgd++, addr = next, addr != end);
     507             : 
     508           0 :         tlb_end_vma(tlb, vma);
     509             : 
     510           0 :         return pages;
     511             : }
     512             : 
     513           0 : long change_protection(struct mmu_gather *tlb,
     514             :                        struct vm_area_struct *vma, unsigned long start,
     515             :                        unsigned long end, unsigned long cp_flags)
     516             : {
     517           0 :         pgprot_t newprot = vma->vm_page_prot;
     518             :         long pages;
     519             : 
     520           0 :         BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL);
     521             : 
     522             : #ifdef CONFIG_NUMA_BALANCING
     523             :         /*
     524             :          * Ordinary protection updates (mprotect, uffd-wp, softdirty tracking)
     525             :          * are expected to reflect their requirements via VMA flags such that
     526             :          * vma_set_page_prot() will adjust vma->vm_page_prot accordingly.
     527             :          */
     528             :         if (cp_flags & MM_CP_PROT_NUMA)
     529             :                 newprot = PAGE_NONE;
     530             : #else
     531           0 :         WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA);
     532             : #endif
     533             : 
     534           0 :         if (is_vm_hugetlb_page(vma))
     535             :                 pages = hugetlb_change_protection(vma, start, end, newprot,
     536             :                                                   cp_flags);
     537             :         else
     538           0 :                 pages = change_protection_range(tlb, vma, start, end, newprot,
     539             :                                                 cp_flags);
     540             : 
     541           0 :         return pages;
     542             : }
     543             : 
     544             : static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
     545             :                                unsigned long next, struct mm_walk *walk)
     546             : {
     547             :         return pfn_modify_allowed(pte_pfn(ptep_get(pte)),
     548             :                                   *(pgprot_t *)(walk->private)) ?
     549             :                 0 : -EACCES;
     550             : }
     551             : 
     552             : static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
     553             :                                    unsigned long addr, unsigned long next,
     554             :                                    struct mm_walk *walk)
     555             : {
     556             :         return pfn_modify_allowed(pte_pfn(ptep_get(pte)),
     557             :                                   *(pgprot_t *)(walk->private)) ?
     558             :                 0 : -EACCES;
     559             : }
     560             : 
     561             : static int prot_none_test(unsigned long addr, unsigned long next,
     562             :                           struct mm_walk *walk)
     563             : {
     564             :         return 0;
     565             : }
     566             : 
     567             : static const struct mm_walk_ops prot_none_walk_ops = {
     568             :         .pte_entry              = prot_none_pte_entry,
     569             :         .hugetlb_entry          = prot_none_hugetlb_entry,
     570             :         .test_walk              = prot_none_test,
     571             : };
     572             : 
     573             : int
     574           0 : mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
     575             :                struct vm_area_struct *vma, struct vm_area_struct **pprev,
     576             :                unsigned long start, unsigned long end, unsigned long newflags)
     577             : {
     578           0 :         struct mm_struct *mm = vma->vm_mm;
     579           0 :         unsigned long oldflags = vma->vm_flags;
     580           0 :         long nrpages = (end - start) >> PAGE_SHIFT;
     581           0 :         unsigned int mm_cp_flags = 0;
     582           0 :         unsigned long charged = 0;
     583             :         pgoff_t pgoff;
     584             :         int error;
     585             : 
     586           0 :         if (newflags == oldflags) {
     587           0 :                 *pprev = vma;
     588           0 :                 return 0;
     589             :         }
     590             : 
     591             :         /*
     592             :          * Do PROT_NONE PFN permission checks here when we can still
     593             :          * bail out without undoing a lot of state. This is a rather
     594             :          * uncommon case, so doesn't need to be very optimized.
     595             :          */
     596             :         if (arch_has_pfn_modify_check() &&
     597             :             (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
     598             :             (newflags & VM_ACCESS_FLAGS) == 0) {
     599             :                 pgprot_t new_pgprot = vm_get_page_prot(newflags);
     600             : 
     601             :                 error = walk_page_range(current->mm, start, end,
     602             :                                 &prot_none_walk_ops, &new_pgprot);
     603             :                 if (error)
     604             :                         return error;
     605             :         }
     606             : 
     607             :         /*
     608             :          * If we make a private mapping writable we increase our commit;
     609             :          * but (without finer accounting) cannot reduce our commit if we
     610             :          * make it unwritable again. hugetlb mapping were accounted for
     611             :          * even if read-only so there is no need to account for them here
     612             :          */
     613           0 :         if (newflags & VM_WRITE) {
     614             :                 /* Check space limits when area turns into data. */
     615           0 :                 if (!may_expand_vm(mm, newflags, nrpages) &&
     616           0 :                                 may_expand_vm(mm, oldflags, nrpages))
     617             :                         return -ENOMEM;
     618           0 :                 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
     619             :                                                 VM_SHARED|VM_NORESERVE))) {
     620           0 :                         charged = nrpages;
     621           0 :                         if (security_vm_enough_memory_mm(mm, charged))
     622             :                                 return -ENOMEM;
     623           0 :                         newflags |= VM_ACCOUNT;
     624             :                 }
     625             :         }
     626             : 
     627             :         /*
     628             :          * First try to merge with previous and/or next vma.
     629             :          */
     630           0 :         pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
     631           0 :         *pprev = vma_merge(vmi, mm, *pprev, start, end, newflags,
     632             :                            vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
     633             :                            vma->vm_userfaultfd_ctx, anon_vma_name(vma));
     634           0 :         if (*pprev) {
     635             :                 vma = *pprev;
     636             :                 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);
     637             :                 goto success;
     638             :         }
     639             : 
     640           0 :         *pprev = vma;
     641             : 
     642           0 :         if (start != vma->vm_start) {
     643           0 :                 error = split_vma(vmi, vma, start, 1);
     644           0 :                 if (error)
     645             :                         goto fail;
     646             :         }
     647             : 
     648           0 :         if (end != vma->vm_end) {
     649           0 :                 error = split_vma(vmi, vma, end, 0);
     650           0 :                 if (error)
     651             :                         goto fail;
     652             :         }
     653             : 
     654             : success:
     655             :         /*
     656             :          * vm_flags and vm_page_prot are protected by the mmap_lock
     657             :          * held in write mode.
     658             :          */
     659           0 :         vm_flags_reset(vma, newflags);
     660           0 :         if (vma_wants_manual_pte_write_upgrade(vma))
     661           0 :                 mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
     662           0 :         vma_set_page_prot(vma);
     663             : 
     664           0 :         change_protection(tlb, vma, start, end, mm_cp_flags);
     665             : 
     666             :         /*
     667             :          * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
     668             :          * fault on access.
     669             :          */
     670           0 :         if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
     671           0 :                         (newflags & VM_WRITE)) {
     672           0 :                 populate_vma_page_range(vma, start, end, NULL);
     673             :         }
     674             : 
     675           0 :         vm_stat_account(mm, oldflags, -nrpages);
     676           0 :         vm_stat_account(mm, newflags, nrpages);
     677           0 :         perf_event_mmap(vma);
     678           0 :         return 0;
     679             : 
     680             : fail:
     681           0 :         vm_unacct_memory(charged);
     682           0 :         return error;
     683             : }
     684             : 
     685             : /*
     686             :  * pkey==-1 when doing a legacy mprotect()
     687             :  */
     688           0 : static int do_mprotect_pkey(unsigned long start, size_t len,
     689             :                 unsigned long prot, int pkey)
     690             : {
     691             :         unsigned long nstart, end, tmp, reqprot;
     692             :         struct vm_area_struct *vma, *prev;
     693             :         int error;
     694           0 :         const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
     695           0 :         const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
     696           0 :                                 (prot & PROT_READ);
     697             :         struct mmu_gather tlb;
     698             :         struct vma_iterator vmi;
     699             : 
     700           0 :         start = untagged_addr(start);
     701             : 
     702           0 :         prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
     703           0 :         if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
     704             :                 return -EINVAL;
     705             : 
     706           0 :         if (start & ~PAGE_MASK)
     707             :                 return -EINVAL;
     708           0 :         if (!len)
     709             :                 return 0;
     710           0 :         len = PAGE_ALIGN(len);
     711           0 :         end = start + len;
     712           0 :         if (end <= start)
     713             :                 return -ENOMEM;
     714           0 :         if (!arch_validate_prot(prot, start))
     715             :                 return -EINVAL;
     716             : 
     717           0 :         reqprot = prot;
     718             : 
     719           0 :         if (mmap_write_lock_killable(current->mm))
     720             :                 return -EINTR;
     721             : 
     722             :         /*
     723             :          * If userspace did not allocate the pkey, do not let
     724             :          * them use it here.
     725             :          */
     726           0 :         error = -EINVAL;
     727           0 :         if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
     728             :                 goto out;
     729             : 
     730           0 :         vma_iter_init(&vmi, current->mm, start);
     731           0 :         vma = vma_find(&vmi, end);
     732           0 :         error = -ENOMEM;
     733           0 :         if (!vma)
     734             :                 goto out;
     735             : 
     736           0 :         if (unlikely(grows & PROT_GROWSDOWN)) {
     737           0 :                 if (vma->vm_start >= end)
     738             :                         goto out;
     739           0 :                 start = vma->vm_start;
     740           0 :                 error = -EINVAL;
     741           0 :                 if (!(vma->vm_flags & VM_GROWSDOWN))
     742             :                         goto out;
     743             :         } else {
     744           0 :                 if (vma->vm_start > start)
     745             :                         goto out;
     746           0 :                 if (unlikely(grows & PROT_GROWSUP)) {
     747             :                         end = vma->vm_end;
     748             :                         error = -EINVAL;
     749             :                         if (!(vma->vm_flags & VM_GROWSUP))
     750             :                                 goto out;
     751             :                 }
     752             :         }
     753             : 
     754           0 :         prev = vma_prev(&vmi);
     755           0 :         if (start > vma->vm_start)
     756           0 :                 prev = vma;
     757             : 
     758           0 :         tlb_gather_mmu(&tlb, current->mm);
     759           0 :         nstart = start;
     760           0 :         tmp = vma->vm_start;
     761           0 :         for_each_vma_range(vmi, vma, end) {
     762             :                 unsigned long mask_off_old_flags;
     763             :                 unsigned long newflags;
     764             :                 int new_vma_pkey;
     765             : 
     766           0 :                 if (vma->vm_start != tmp) {
     767             :                         error = -ENOMEM;
     768             :                         break;
     769             :                 }
     770             : 
     771             :                 /* Does the application expect PROT_READ to imply PROT_EXEC */
     772           0 :                 if (rier && (vma->vm_flags & VM_MAYEXEC))
     773           0 :                         prot |= PROT_EXEC;
     774             : 
     775             :                 /*
     776             :                  * Each mprotect() call explicitly passes r/w/x permissions.
     777             :                  * If a permission is not passed to mprotect(), it must be
     778             :                  * cleared from the VMA.
     779             :                  */
     780           0 :                 mask_off_old_flags = VM_ACCESS_FLAGS | VM_FLAGS_CLEAR;
     781             : 
     782           0 :                 new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
     783           0 :                 newflags = calc_vm_prot_bits(prot, new_vma_pkey);
     784           0 :                 newflags |= (vma->vm_flags & ~mask_off_old_flags);
     785             : 
     786             :                 /* newflags >> 4 shift VM_MAY% in place of VM_% */
     787           0 :                 if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) {
     788             :                         error = -EACCES;
     789             :                         break;
     790             :                 }
     791             : 
     792           0 :                 if (map_deny_write_exec(vma, newflags)) {
     793             :                         error = -EACCES;
     794             :                         break;
     795             :                 }
     796             : 
     797             :                 /* Allow architectures to sanity-check the new flags */
     798           0 :                 if (!arch_validate_flags(newflags)) {
     799             :                         error = -EINVAL;
     800             :                         break;
     801             :                 }
     802             : 
     803           0 :                 error = security_file_mprotect(vma, reqprot, prot);
     804             :                 if (error)
     805             :                         break;
     806             : 
     807           0 :                 tmp = vma->vm_end;
     808           0 :                 if (tmp > end)
     809           0 :                         tmp = end;
     810             : 
     811           0 :                 if (vma->vm_ops && vma->vm_ops->mprotect) {
     812           0 :                         error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags);
     813           0 :                         if (error)
     814             :                                 break;
     815             :                 }
     816             : 
     817           0 :                 error = mprotect_fixup(&vmi, &tlb, vma, &prev, nstart, tmp, newflags);
     818           0 :                 if (error)
     819             :                         break;
     820             : 
     821           0 :                 tmp = vma_iter_end(&vmi);
     822           0 :                 nstart = tmp;
     823           0 :                 prot = reqprot;
     824             :         }
     825           0 :         tlb_finish_mmu(&tlb);
     826             : 
     827           0 :         if (!error && tmp < end)
     828           0 :                 error = -ENOMEM;
     829             : 
     830             : out:
     831           0 :         mmap_write_unlock(current->mm);
     832           0 :         return error;
     833             : }
     834             : 
     835           0 : SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
     836             :                 unsigned long, prot)
     837             : {
     838           0 :         return do_mprotect_pkey(start, len, prot, -1);
     839             : }
     840             : 
     841             : #ifdef CONFIG_ARCH_HAS_PKEYS
     842             : 
     843             : SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
     844             :                 unsigned long, prot, int, pkey)
     845             : {
     846             :         return do_mprotect_pkey(start, len, prot, pkey);
     847             : }
     848             : 
     849             : SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
     850             : {
     851             :         int pkey;
     852             :         int ret;
     853             : 
     854             :         /* No flags supported yet. */
     855             :         if (flags)
     856             :                 return -EINVAL;
     857             :         /* check for unsupported init values */
     858             :         if (init_val & ~PKEY_ACCESS_MASK)
     859             :                 return -EINVAL;
     860             : 
     861             :         mmap_write_lock(current->mm);
     862             :         pkey = mm_pkey_alloc(current->mm);
     863             : 
     864             :         ret = -ENOSPC;
     865             :         if (pkey == -1)
     866             :                 goto out;
     867             : 
     868             :         ret = arch_set_user_pkey_access(current, pkey, init_val);
     869             :         if (ret) {
     870             :                 mm_pkey_free(current->mm, pkey);
     871             :                 goto out;
     872             :         }
     873             :         ret = pkey;
     874             : out:
     875             :         mmap_write_unlock(current->mm);
     876             :         return ret;
     877             : }
     878             : 
     879             : SYSCALL_DEFINE1(pkey_free, int, pkey)
     880             : {
     881             :         int ret;
     882             : 
     883             :         mmap_write_lock(current->mm);
     884             :         ret = mm_pkey_free(current->mm, pkey);
     885             :         mmap_write_unlock(current->mm);
     886             : 
     887             :         /*
     888             :          * We could provide warnings or errors if any VMA still
     889             :          * has the pkey set here.
     890             :          */
     891             :         return ret;
     892             : }
     893             : 
     894             : #endif /* CONFIG_ARCH_HAS_PKEYS */

Generated by: LCOV version 1.14