Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * mm/pgtable-generic.c
4 : *
5 : * Generic pgtable methods declared in linux/pgtable.h
6 : *
7 : * Copyright (C) 2010 Linus Torvalds
8 : */
9 :
10 : #include <linux/pagemap.h>
11 : #include <linux/hugetlb.h>
12 : #include <linux/pgtable.h>
13 : #include <linux/swap.h>
14 : #include <linux/swapops.h>
15 : #include <linux/mm_inline.h>
16 : #include <asm/tlb.h>
17 :
18 : /*
19 : * If a p?d_bad entry is found while walking page tables, report
20 : * the error, before resetting entry to p?d_none. Usually (but
21 : * very seldom) called out from the p?d_none_or_clear_bad macros.
22 : */
23 :
24 0 : void pgd_clear_bad(pgd_t *pgd)
25 : {
26 0 : pgd_ERROR(*pgd);
27 0 : pgd_clear(pgd);
28 0 : }
29 :
30 : #ifndef __PAGETABLE_P4D_FOLDED
31 : void p4d_clear_bad(p4d_t *p4d)
32 : {
33 : p4d_ERROR(*p4d);
34 : p4d_clear(p4d);
35 : }
36 : #endif
37 :
38 : #ifndef __PAGETABLE_PUD_FOLDED
39 : void pud_clear_bad(pud_t *pud)
40 : {
41 : pud_ERROR(*pud);
42 : pud_clear(pud);
43 : }
44 : #endif
45 :
46 : /*
47 : * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
48 : * above. pmd folding is special and typically pmd_* macros refer to upper
49 : * level even when folded
50 : */
51 0 : void pmd_clear_bad(pmd_t *pmd)
52 : {
53 0 : pmd_ERROR(*pmd);
54 0 : pmd_clear(pmd);
55 0 : }
56 :
57 : #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
58 : /*
59 : * Only sets the access flags (dirty, accessed), as well as write
60 : * permission. Furthermore, we know it always gets set to a "more
61 : * permissive" setting, which allows most architectures to optimize
62 : * this. We return whether the PTE actually changed, which in turn
63 : * instructs the caller to do things like update__mmu_cache. This
64 : * used to be done in the caller, but sparc needs minor faults to
65 : * force that call on sun4c so we changed this macro slightly
66 : */
67 0 : int ptep_set_access_flags(struct vm_area_struct *vma,
68 : unsigned long address, pte_t *ptep,
69 : pte_t entry, int dirty)
70 : {
71 0 : int changed = !pte_same(ptep_get(ptep), entry);
72 0 : if (changed) {
73 0 : set_pte_at(vma->vm_mm, address, ptep, entry);
74 0 : flush_tlb_fix_spurious_fault(vma, address, ptep);
75 : }
76 0 : return changed;
77 : }
78 : #endif
79 :
80 : #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
81 0 : int ptep_clear_flush_young(struct vm_area_struct *vma,
82 : unsigned long address, pte_t *ptep)
83 : {
84 : int young;
85 0 : young = ptep_test_and_clear_young(vma, address, ptep);
86 0 : if (young)
87 0 : flush_tlb_page(vma, address);
88 0 : return young;
89 : }
90 : #endif
91 :
92 : #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
93 0 : pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
94 : pte_t *ptep)
95 : {
96 0 : struct mm_struct *mm = (vma)->vm_mm;
97 : pte_t pte;
98 0 : pte = ptep_get_and_clear(mm, address, ptep);
99 : if (pte_accessible(mm, pte))
100 0 : flush_tlb_page(vma, address);
101 0 : return pte;
102 : }
103 : #endif
104 :
105 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
106 :
107 : #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
108 : int pmdp_set_access_flags(struct vm_area_struct *vma,
109 : unsigned long address, pmd_t *pmdp,
110 : pmd_t entry, int dirty)
111 : {
112 : int changed = !pmd_same(*pmdp, entry);
113 : VM_BUG_ON(address & ~HPAGE_PMD_MASK);
114 : if (changed) {
115 : set_pmd_at(vma->vm_mm, address, pmdp, entry);
116 : flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
117 : }
118 : return changed;
119 : }
120 : #endif
121 :
122 : #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
123 : int pmdp_clear_flush_young(struct vm_area_struct *vma,
124 : unsigned long address, pmd_t *pmdp)
125 : {
126 : int young;
127 : VM_BUG_ON(address & ~HPAGE_PMD_MASK);
128 : young = pmdp_test_and_clear_young(vma, address, pmdp);
129 : if (young)
130 : flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
131 : return young;
132 : }
133 : #endif
134 :
135 : #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
136 : pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
137 : pmd_t *pmdp)
138 : {
139 : pmd_t pmd;
140 : VM_BUG_ON(address & ~HPAGE_PMD_MASK);
141 : VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) &&
142 : !pmd_devmap(*pmdp));
143 : pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
144 : flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
145 : return pmd;
146 : }
147 :
148 : #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
149 : pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
150 : pud_t *pudp)
151 : {
152 : pud_t pud;
153 :
154 : VM_BUG_ON(address & ~HPAGE_PUD_MASK);
155 : VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp));
156 : pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
157 : flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);
158 : return pud;
159 : }
160 : #endif
161 : #endif
162 :
163 : #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
164 : void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
165 : pgtable_t pgtable)
166 : {
167 : assert_spin_locked(pmd_lockptr(mm, pmdp));
168 :
169 : /* FIFO */
170 : if (!pmd_huge_pte(mm, pmdp))
171 : INIT_LIST_HEAD(&pgtable->lru);
172 : else
173 : list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
174 : pmd_huge_pte(mm, pmdp) = pgtable;
175 : }
176 : #endif
177 :
178 : #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
179 : /* no "address" argument so destroys page coloring of some arch */
180 : pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
181 : {
182 : pgtable_t pgtable;
183 :
184 : assert_spin_locked(pmd_lockptr(mm, pmdp));
185 :
186 : /* FIFO */
187 : pgtable = pmd_huge_pte(mm, pmdp);
188 : pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
189 : struct page, lru);
190 : if (pmd_huge_pte(mm, pmdp))
191 : list_del(&pgtable->lru);
192 : return pgtable;
193 : }
194 : #endif
195 :
196 : #ifndef __HAVE_ARCH_PMDP_INVALIDATE
197 : pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
198 : pmd_t *pmdp)
199 : {
200 : pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp));
201 : flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
202 : return old;
203 : }
204 : #endif
205 :
206 : #ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
207 : pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
208 : pmd_t *pmdp)
209 : {
210 : return pmdp_invalidate(vma, address, pmdp);
211 : }
212 : #endif
213 :
214 : #ifndef pmdp_collapse_flush
215 : pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
216 : pmd_t *pmdp)
217 : {
218 : /*
219 : * pmd and hugepage pte format are same. So we could
220 : * use the same function.
221 : */
222 : pmd_t pmd;
223 :
224 : VM_BUG_ON(address & ~HPAGE_PMD_MASK);
225 : VM_BUG_ON(pmd_trans_huge(*pmdp));
226 : pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
227 :
228 : /* collapse entails shooting down ptes not pmd */
229 : flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
230 : return pmd;
231 : }
232 : #endif
233 : #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
234 :
235 0 : pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
236 : {
237 : pmd_t pmdval;
238 :
239 : /* rcu_read_lock() to be added later */
240 0 : pmdval = pmdp_get_lockless(pmd);
241 0 : if (pmdvalp)
242 0 : *pmdvalp = pmdval;
243 0 : if (unlikely(pmd_none(pmdval) || is_pmd_migration_entry(pmdval)))
244 : goto nomap;
245 0 : if (unlikely(pmd_trans_huge(pmdval) || pmd_devmap(pmdval)))
246 : goto nomap;
247 0 : if (unlikely(pmd_bad(pmdval))) {
248 : pmd_clear_bad(pmd);
249 : goto nomap;
250 : }
251 0 : return __pte_map(&pmdval, addr);
252 : nomap:
253 : /* rcu_read_unlock() to be added later */
254 : return NULL;
255 : }
256 :
257 0 : pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
258 : unsigned long addr, spinlock_t **ptlp)
259 : {
260 : pmd_t pmdval;
261 : pte_t *pte;
262 :
263 0 : pte = __pte_offset_map(pmd, addr, &pmdval);
264 0 : if (likely(pte))
265 0 : *ptlp = pte_lockptr(mm, &pmdval);
266 0 : return pte;
267 : }
268 :
269 0 : pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
270 : unsigned long addr, spinlock_t **ptlp)
271 : {
272 : spinlock_t *ptl;
273 : pmd_t pmdval;
274 : pte_t *pte;
275 : again:
276 0 : pte = __pte_offset_map(pmd, addr, &pmdval);
277 0 : if (unlikely(!pte))
278 : return pte;
279 0 : ptl = pte_lockptr(mm, &pmdval);
280 0 : spin_lock(ptl);
281 0 : if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
282 0 : *ptlp = ptl;
283 0 : return pte;
284 : }
285 0 : pte_unmap_unlock(pte, ptl);
286 : goto again;
287 : }
|
