Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : #include <linux/pagewalk.h>
3 : #include <linux/mm_inline.h>
4 : #include <linux/hugetlb.h>
5 : #include <linux/huge_mm.h>
6 : #include <linux/mount.h>
7 : #include <linux/seq_file.h>
8 : #include <linux/highmem.h>
9 : #include <linux/ptrace.h>
10 : #include <linux/slab.h>
11 : #include <linux/pagemap.h>
12 : #include <linux/mempolicy.h>
13 : #include <linux/rmap.h>
14 : #include <linux/swap.h>
15 : #include <linux/sched/mm.h>
16 : #include <linux/swapops.h>
17 : #include <linux/mmu_notifier.h>
18 : #include <linux/page_idle.h>
19 : #include <linux/shmem_fs.h>
20 : #include <linux/uaccess.h>
21 : #include <linux/pkeys.h>
22 :
23 : #include <asm/elf.h>
24 : #include <asm/tlb.h>
25 : #include <asm/tlbflush.h>
26 : #include "internal.h"
27 :
28 : #define SEQ_PUT_DEC(str, val) \
29 : seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
30 0 : void task_mem(struct seq_file *m, struct mm_struct *mm)
31 : {
32 : unsigned long text, lib, swap, anon, file, shmem;
33 : unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
34 :
35 0 : anon = get_mm_counter(mm, MM_ANONPAGES);
36 0 : file = get_mm_counter(mm, MM_FILEPAGES);
37 0 : shmem = get_mm_counter(mm, MM_SHMEMPAGES);
38 :
39 : /*
40 : * Note: to minimize their overhead, mm maintains hiwater_vm and
41 : * hiwater_rss only when about to *lower* total_vm or rss. Any
42 : * collector of these hiwater stats must therefore get total_vm
43 : * and rss too, which will usually be the higher. Barriers? not
44 : * worth the effort, such snapshots can always be inconsistent.
45 : */
46 0 : hiwater_vm = total_vm = mm->total_vm;
47 0 : if (hiwater_vm < mm->hiwater_vm)
48 0 : hiwater_vm = mm->hiwater_vm;
49 0 : hiwater_rss = total_rss = anon + file + shmem;
50 0 : if (hiwater_rss < mm->hiwater_rss)
51 0 : hiwater_rss = mm->hiwater_rss;
52 :
53 : /* split executable areas between text and lib */
54 0 : text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
55 0 : text = min(text, mm->exec_vm << PAGE_SHIFT);
56 0 : lib = (mm->exec_vm << PAGE_SHIFT) - text;
57 :
58 0 : swap = get_mm_counter(mm, MM_SWAPENTS);
59 0 : SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
60 0 : SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
61 0 : SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
62 0 : SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
63 0 : SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
64 0 : SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
65 0 : SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
66 0 : SEQ_PUT_DEC(" kB\nRssFile:\t", file);
67 0 : SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
68 0 : SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
69 0 : SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
70 0 : seq_put_decimal_ull_width(m,
71 0 : " kB\nVmExe:\t", text >> 10, 8);
72 0 : seq_put_decimal_ull_width(m,
73 0 : " kB\nVmLib:\t", lib >> 10, 8);
74 0 : seq_put_decimal_ull_width(m,
75 0 : " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
76 0 : SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
77 0 : seq_puts(m, " kB\n");
78 0 : hugetlb_report_usage(m, mm);
79 0 : }
80 : #undef SEQ_PUT_DEC
81 :
82 0 : unsigned long task_vsize(struct mm_struct *mm)
83 : {
84 0 : return PAGE_SIZE * mm->total_vm;
85 : }
86 :
87 0 : unsigned long task_statm(struct mm_struct *mm,
88 : unsigned long *shared, unsigned long *text,
89 : unsigned long *data, unsigned long *resident)
90 : {
91 0 : *shared = get_mm_counter(mm, MM_FILEPAGES) +
92 0 : get_mm_counter(mm, MM_SHMEMPAGES);
93 0 : *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
94 0 : >> PAGE_SHIFT;
95 0 : *data = mm->data_vm + mm->stack_vm;
96 0 : *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
97 0 : return mm->total_vm;
98 : }
99 :
100 : #ifdef CONFIG_NUMA
101 : /*
102 : * Save get_task_policy() for show_numa_map().
103 : */
104 : static void hold_task_mempolicy(struct proc_maps_private *priv)
105 : {
106 : struct task_struct *task = priv->task;
107 :
108 : task_lock(task);
109 : priv->task_mempolicy = get_task_policy(task);
110 : mpol_get(priv->task_mempolicy);
111 : task_unlock(task);
112 : }
113 : static void release_task_mempolicy(struct proc_maps_private *priv)
114 : {
115 : mpol_put(priv->task_mempolicy);
116 : }
117 : #else
118 : static void hold_task_mempolicy(struct proc_maps_private *priv)
119 : {
120 : }
121 : static void release_task_mempolicy(struct proc_maps_private *priv)
122 : {
123 : }
124 : #endif
125 :
126 : static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv,
127 : loff_t *ppos)
128 : {
129 0 : struct vm_area_struct *vma = vma_next(&priv->iter);
130 :
131 0 : if (vma) {
132 0 : *ppos = vma->vm_start;
133 : } else {
134 0 : *ppos = -2UL;
135 0 : vma = get_gate_vma(priv->mm);
136 : }
137 :
138 : return vma;
139 : }
140 :
141 0 : static void *m_start(struct seq_file *m, loff_t *ppos)
142 : {
143 0 : struct proc_maps_private *priv = m->private;
144 0 : unsigned long last_addr = *ppos;
145 : struct mm_struct *mm;
146 :
147 : /* See m_next(). Zero at the start or after lseek. */
148 0 : if (last_addr == -1UL)
149 : return NULL;
150 :
151 0 : priv->task = get_proc_task(priv->inode);
152 0 : if (!priv->task)
153 : return ERR_PTR(-ESRCH);
154 :
155 0 : mm = priv->mm;
156 0 : if (!mm || !mmget_not_zero(mm)) {
157 0 : put_task_struct(priv->task);
158 0 : priv->task = NULL;
159 0 : return NULL;
160 : }
161 :
162 0 : if (mmap_read_lock_killable(mm)) {
163 0 : mmput(mm);
164 0 : put_task_struct(priv->task);
165 0 : priv->task = NULL;
166 0 : return ERR_PTR(-EINTR);
167 : }
168 :
169 0 : vma_iter_init(&priv->iter, mm, last_addr);
170 0 : hold_task_mempolicy(priv);
171 0 : if (last_addr == -2UL)
172 : return get_gate_vma(mm);
173 :
174 : return proc_get_vma(priv, ppos);
175 : }
176 :
177 0 : static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
178 : {
179 0 : if (*ppos == -2UL) {
180 0 : *ppos = -1UL;
181 0 : return NULL;
182 : }
183 0 : return proc_get_vma(m->private, ppos);
184 : }
185 :
186 0 : static void m_stop(struct seq_file *m, void *v)
187 : {
188 0 : struct proc_maps_private *priv = m->private;
189 0 : struct mm_struct *mm = priv->mm;
190 :
191 0 : if (!priv->task)
192 : return;
193 :
194 0 : release_task_mempolicy(priv);
195 0 : mmap_read_unlock(mm);
196 0 : mmput(mm);
197 0 : put_task_struct(priv->task);
198 0 : priv->task = NULL;
199 : }
200 :
201 0 : static int proc_maps_open(struct inode *inode, struct file *file,
202 : const struct seq_operations *ops, int psize)
203 : {
204 0 : struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
205 :
206 0 : if (!priv)
207 : return -ENOMEM;
208 :
209 0 : priv->inode = inode;
210 0 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
211 0 : if (IS_ERR(priv->mm)) {
212 0 : int err = PTR_ERR(priv->mm);
213 :
214 0 : seq_release_private(inode, file);
215 0 : return err;
216 : }
217 :
218 : return 0;
219 : }
220 :
221 0 : static int proc_map_release(struct inode *inode, struct file *file)
222 : {
223 0 : struct seq_file *seq = file->private_data;
224 0 : struct proc_maps_private *priv = seq->private;
225 :
226 0 : if (priv->mm)
227 0 : mmdrop(priv->mm);
228 :
229 0 : return seq_release_private(inode, file);
230 : }
231 :
232 : static int do_maps_open(struct inode *inode, struct file *file,
233 : const struct seq_operations *ops)
234 : {
235 0 : return proc_maps_open(inode, file, ops,
236 : sizeof(struct proc_maps_private));
237 : }
238 :
239 : /*
240 : * Indicate if the VMA is a stack for the given task; for
241 : * /proc/PID/maps that is the stack of the main task.
242 : */
243 : static int is_stack(struct vm_area_struct *vma)
244 : {
245 : /*
246 : * We make no effort to guess what a given thread considers to be
247 : * its "stack". It's not even well-defined for programs written
248 : * languages like Go.
249 : */
250 0 : return vma->vm_start <= vma->vm_mm->start_stack &&
251 0 : vma->vm_end >= vma->vm_mm->start_stack;
252 : }
253 :
254 0 : static void show_vma_header_prefix(struct seq_file *m,
255 : unsigned long start, unsigned long end,
256 : vm_flags_t flags, unsigned long long pgoff,
257 : dev_t dev, unsigned long ino)
258 : {
259 0 : seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
260 0 : seq_put_hex_ll(m, NULL, start, 8);
261 0 : seq_put_hex_ll(m, "-", end, 8);
262 0 : seq_putc(m, ' ');
263 0 : seq_putc(m, flags & VM_READ ? 'r' : '-');
264 0 : seq_putc(m, flags & VM_WRITE ? 'w' : '-');
265 0 : seq_putc(m, flags & VM_EXEC ? 'x' : '-');
266 0 : seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
267 0 : seq_put_hex_ll(m, " ", pgoff, 8);
268 0 : seq_put_hex_ll(m, " ", MAJOR(dev), 2);
269 0 : seq_put_hex_ll(m, ":", MINOR(dev), 2);
270 0 : seq_put_decimal_ull(m, " ", ino);
271 0 : seq_putc(m, ' ');
272 0 : }
273 :
274 : static void
275 0 : show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
276 : {
277 0 : struct anon_vma_name *anon_name = NULL;
278 0 : struct mm_struct *mm = vma->vm_mm;
279 0 : struct file *file = vma->vm_file;
280 0 : vm_flags_t flags = vma->vm_flags;
281 0 : unsigned long ino = 0;
282 0 : unsigned long long pgoff = 0;
283 : unsigned long start, end;
284 0 : dev_t dev = 0;
285 0 : const char *name = NULL;
286 :
287 0 : if (file) {
288 0 : struct inode *inode = file_inode(vma->vm_file);
289 0 : dev = inode->i_sb->s_dev;
290 0 : ino = inode->i_ino;
291 0 : pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
292 : }
293 :
294 0 : start = vma->vm_start;
295 0 : end = vma->vm_end;
296 0 : show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
297 : if (mm)
298 : anon_name = anon_vma_name(vma);
299 :
300 : /*
301 : * Print the dentry name for named mappings, and a
302 : * special [heap] marker for the heap:
303 : */
304 0 : if (file) {
305 0 : seq_pad(m, ' ');
306 : /*
307 : * If user named this anon shared memory via
308 : * prctl(PR_SET_VMA ..., use the provided name.
309 : */
310 : if (anon_name)
311 : seq_printf(m, "[anon_shmem:%s]", anon_name->name);
312 : else
313 0 : seq_file_path(m, file, "\n");
314 : goto done;
315 : }
316 :
317 0 : if (vma->vm_ops && vma->vm_ops->name) {
318 0 : name = vma->vm_ops->name(vma);
319 0 : if (name)
320 : goto done;
321 : }
322 :
323 0 : name = arch_vma_name(vma);
324 0 : if (!name) {
325 0 : if (!mm) {
326 : name = "[vdso]";
327 : goto done;
328 : }
329 :
330 0 : if (vma->vm_start <= mm->brk &&
331 0 : vma->vm_end >= mm->start_brk) {
332 : name = "[heap]";
333 : goto done;
334 : }
335 :
336 0 : if (is_stack(vma)) {
337 : name = "[stack]";
338 : goto done;
339 : }
340 :
341 : if (anon_name) {
342 : seq_pad(m, ' ');
343 : seq_printf(m, "[anon:%s]", anon_name->name);
344 : }
345 : }
346 :
347 : done:
348 0 : if (name) {
349 0 : seq_pad(m, ' ');
350 0 : seq_puts(m, name);
351 : }
352 0 : seq_putc(m, '\n');
353 0 : }
354 :
355 0 : static int show_map(struct seq_file *m, void *v)
356 : {
357 0 : show_map_vma(m, v);
358 0 : return 0;
359 : }
360 :
361 : static const struct seq_operations proc_pid_maps_op = {
362 : .start = m_start,
363 : .next = m_next,
364 : .stop = m_stop,
365 : .show = show_map
366 : };
367 :
368 0 : static int pid_maps_open(struct inode *inode, struct file *file)
369 : {
370 0 : return do_maps_open(inode, file, &proc_pid_maps_op);
371 : }
372 :
373 : const struct file_operations proc_pid_maps_operations = {
374 : .open = pid_maps_open,
375 : .read = seq_read,
376 : .llseek = seq_lseek,
377 : .release = proc_map_release,
378 : };
379 :
380 : /*
381 : * Proportional Set Size(PSS): my share of RSS.
382 : *
383 : * PSS of a process is the count of pages it has in memory, where each
384 : * page is divided by the number of processes sharing it. So if a
385 : * process has 1000 pages all to itself, and 1000 shared with one other
386 : * process, its PSS will be 1500.
387 : *
388 : * To keep (accumulated) division errors low, we adopt a 64bit
389 : * fixed-point pss counter to minimize division errors. So (pss >>
390 : * PSS_SHIFT) would be the real byte count.
391 : *
392 : * A shift of 12 before division means (assuming 4K page size):
393 : * - 1M 3-user-pages add up to 8KB errors;
394 : * - supports mapcount up to 2^24, or 16M;
395 : * - supports PSS up to 2^52 bytes, or 4PB.
396 : */
397 : #define PSS_SHIFT 12
398 :
399 : #ifdef CONFIG_PROC_PAGE_MONITOR
400 : struct mem_size_stats {
401 : unsigned long resident;
402 : unsigned long shared_clean;
403 : unsigned long shared_dirty;
404 : unsigned long private_clean;
405 : unsigned long private_dirty;
406 : unsigned long referenced;
407 : unsigned long anonymous;
408 : unsigned long lazyfree;
409 : unsigned long anonymous_thp;
410 : unsigned long shmem_thp;
411 : unsigned long file_thp;
412 : unsigned long swap;
413 : unsigned long shared_hugetlb;
414 : unsigned long private_hugetlb;
415 : u64 pss;
416 : u64 pss_anon;
417 : u64 pss_file;
418 : u64 pss_shmem;
419 : u64 pss_dirty;
420 : u64 pss_locked;
421 : u64 swap_pss;
422 : };
423 :
424 0 : static void smaps_page_accumulate(struct mem_size_stats *mss,
425 : struct page *page, unsigned long size, unsigned long pss,
426 : bool dirty, bool locked, bool private)
427 : {
428 0 : mss->pss += pss;
429 :
430 0 : if (PageAnon(page))
431 0 : mss->pss_anon += pss;
432 0 : else if (PageSwapBacked(page))
433 0 : mss->pss_shmem += pss;
434 : else
435 0 : mss->pss_file += pss;
436 :
437 0 : if (locked)
438 0 : mss->pss_locked += pss;
439 :
440 0 : if (dirty || PageDirty(page)) {
441 0 : mss->pss_dirty += pss;
442 0 : if (private)
443 0 : mss->private_dirty += size;
444 : else
445 0 : mss->shared_dirty += size;
446 : } else {
447 0 : if (private)
448 0 : mss->private_clean += size;
449 : else
450 0 : mss->shared_clean += size;
451 : }
452 0 : }
453 :
454 0 : static void smaps_account(struct mem_size_stats *mss, struct page *page,
455 : bool compound, bool young, bool dirty, bool locked,
456 : bool migration)
457 : {
458 0 : int i, nr = compound ? compound_nr(page) : 1;
459 0 : unsigned long size = nr * PAGE_SIZE;
460 :
461 : /*
462 : * First accumulate quantities that depend only on |size| and the type
463 : * of the compound page.
464 : */
465 0 : if (PageAnon(page)) {
466 0 : mss->anonymous += size;
467 0 : if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
468 0 : mss->lazyfree += size;
469 : }
470 :
471 0 : mss->resident += size;
472 : /* Accumulate the size in pages that have been accessed. */
473 0 : if (young || page_is_young(page) || PageReferenced(page))
474 0 : mss->referenced += size;
475 :
476 : /*
477 : * Then accumulate quantities that may depend on sharing, or that may
478 : * differ page-by-page.
479 : *
480 : * page_count(page) == 1 guarantees the page is mapped exactly once.
481 : * If any subpage of the compound page mapped with PTE it would elevate
482 : * page_count().
483 : *
484 : * The page_mapcount() is called to get a snapshot of the mapcount.
485 : * Without holding the page lock this snapshot can be slightly wrong as
486 : * we cannot always read the mapcount atomically. It is not safe to
487 : * call page_mapcount() even with PTL held if the page is not mapped,
488 : * especially for migration entries. Treat regular migration entries
489 : * as mapcount == 1.
490 : */
491 0 : if ((page_count(page) == 1) || migration) {
492 0 : smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty,
493 : locked, true);
494 0 : return;
495 : }
496 0 : for (i = 0; i < nr; i++, page++) {
497 0 : int mapcount = page_mapcount(page);
498 0 : unsigned long pss = PAGE_SIZE << PSS_SHIFT;
499 0 : if (mapcount >= 2)
500 0 : pss /= mapcount;
501 0 : smaps_page_accumulate(mss, page, PAGE_SIZE, pss, dirty, locked,
502 : mapcount < 2);
503 : }
504 : }
505 :
506 : #ifdef CONFIG_SHMEM
507 0 : static int smaps_pte_hole(unsigned long addr, unsigned long end,
508 : __always_unused int depth, struct mm_walk *walk)
509 : {
510 0 : struct mem_size_stats *mss = walk->private;
511 0 : struct vm_area_struct *vma = walk->vma;
512 :
513 0 : mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping,
514 : linear_page_index(vma, addr),
515 : linear_page_index(vma, end));
516 :
517 0 : return 0;
518 : }
519 : #else
520 : #define smaps_pte_hole NULL
521 : #endif /* CONFIG_SHMEM */
522 :
523 : static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk)
524 : {
525 : #ifdef CONFIG_SHMEM
526 0 : if (walk->ops->pte_hole) {
527 : /* depth is not used */
528 0 : smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk);
529 : }
530 : #endif
531 : }
532 :
533 0 : static void smaps_pte_entry(pte_t *pte, unsigned long addr,
534 : struct mm_walk *walk)
535 : {
536 0 : struct mem_size_stats *mss = walk->private;
537 0 : struct vm_area_struct *vma = walk->vma;
538 0 : bool locked = !!(vma->vm_flags & VM_LOCKED);
539 0 : struct page *page = NULL;
540 0 : bool migration = false, young = false, dirty = false;
541 0 : pte_t ptent = ptep_get(pte);
542 :
543 0 : if (pte_present(ptent)) {
544 0 : page = vm_normal_page(vma, addr, ptent);
545 0 : young = pte_young(ptent);
546 0 : dirty = pte_dirty(ptent);
547 0 : } else if (is_swap_pte(ptent)) {
548 0 : swp_entry_t swpent = pte_to_swp_entry(ptent);
549 :
550 0 : if (!non_swap_entry(swpent)) {
551 : int mapcount;
552 :
553 0 : mss->swap += PAGE_SIZE;
554 0 : mapcount = swp_swapcount(swpent);
555 0 : if (mapcount >= 2) {
556 0 : u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
557 :
558 0 : do_div(pss_delta, mapcount);
559 0 : mss->swap_pss += pss_delta;
560 : } else {
561 0 : mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
562 : }
563 0 : } else if (is_pfn_swap_entry(swpent)) {
564 0 : if (is_migration_entry(swpent))
565 0 : migration = true;
566 0 : page = pfn_swap_entry_to_page(swpent);
567 : }
568 : } else {
569 : smaps_pte_hole_lookup(addr, walk);
570 : return;
571 : }
572 :
573 0 : if (!page)
574 : return;
575 :
576 0 : smaps_account(mss, page, false, young, dirty, locked, migration);
577 : }
578 :
579 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
580 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
581 : struct mm_walk *walk)
582 : {
583 : struct mem_size_stats *mss = walk->private;
584 : struct vm_area_struct *vma = walk->vma;
585 : bool locked = !!(vma->vm_flags & VM_LOCKED);
586 : struct page *page = NULL;
587 : bool migration = false;
588 :
589 : if (pmd_present(*pmd)) {
590 : /* FOLL_DUMP will return -EFAULT on huge zero page */
591 : page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
592 : } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) {
593 : swp_entry_t entry = pmd_to_swp_entry(*pmd);
594 :
595 : if (is_migration_entry(entry)) {
596 : migration = true;
597 : page = pfn_swap_entry_to_page(entry);
598 : }
599 : }
600 : if (IS_ERR_OR_NULL(page))
601 : return;
602 : if (PageAnon(page))
603 : mss->anonymous_thp += HPAGE_PMD_SIZE;
604 : else if (PageSwapBacked(page))
605 : mss->shmem_thp += HPAGE_PMD_SIZE;
606 : else if (is_zone_device_page(page))
607 : /* pass */;
608 : else
609 : mss->file_thp += HPAGE_PMD_SIZE;
610 :
611 : smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd),
612 : locked, migration);
613 : }
614 : #else
615 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
616 : struct mm_walk *walk)
617 : {
618 : }
619 : #endif
620 :
621 0 : static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
622 : struct mm_walk *walk)
623 : {
624 0 : struct vm_area_struct *vma = walk->vma;
625 : pte_t *pte;
626 : spinlock_t *ptl;
627 :
628 0 : ptl = pmd_trans_huge_lock(pmd, vma);
629 : if (ptl) {
630 : smaps_pmd_entry(pmd, addr, walk);
631 : spin_unlock(ptl);
632 : goto out;
633 : }
634 :
635 0 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
636 0 : if (!pte) {
637 0 : walk->action = ACTION_AGAIN;
638 0 : return 0;
639 : }
640 0 : for (; addr != end; pte++, addr += PAGE_SIZE)
641 0 : smaps_pte_entry(pte, addr, walk);
642 0 : pte_unmap_unlock(pte - 1, ptl);
643 : out:
644 0 : cond_resched();
645 0 : return 0;
646 : }
647 :
648 0 : static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
649 : {
650 : /*
651 : * Don't forget to update Documentation/ on changes.
652 : */
653 : static const char mnemonics[BITS_PER_LONG][2] = {
654 : /*
655 : * In case if we meet a flag we don't know about.
656 : */
657 : [0 ... (BITS_PER_LONG-1)] = "??",
658 :
659 : [ilog2(VM_READ)] = "rd",
660 : [ilog2(VM_WRITE)] = "wr",
661 : [ilog2(VM_EXEC)] = "ex",
662 : [ilog2(VM_SHARED)] = "sh",
663 : [ilog2(VM_MAYREAD)] = "mr",
664 : [ilog2(VM_MAYWRITE)] = "mw",
665 : [ilog2(VM_MAYEXEC)] = "me",
666 : [ilog2(VM_MAYSHARE)] = "ms",
667 : [ilog2(VM_GROWSDOWN)] = "gd",
668 : [ilog2(VM_PFNMAP)] = "pf",
669 : [ilog2(VM_LOCKED)] = "lo",
670 : [ilog2(VM_IO)] = "io",
671 : [ilog2(VM_SEQ_READ)] = "sr",
672 : [ilog2(VM_RAND_READ)] = "rr",
673 : [ilog2(VM_DONTCOPY)] = "dc",
674 : [ilog2(VM_DONTEXPAND)] = "de",
675 : [ilog2(VM_LOCKONFAULT)] = "lf",
676 : [ilog2(VM_ACCOUNT)] = "ac",
677 : [ilog2(VM_NORESERVE)] = "nr",
678 : [ilog2(VM_HUGETLB)] = "ht",
679 : [ilog2(VM_SYNC)] = "sf",
680 : [ilog2(VM_ARCH_1)] = "ar",
681 : [ilog2(VM_WIPEONFORK)] = "wf",
682 : [ilog2(VM_DONTDUMP)] = "dd",
683 : #ifdef CONFIG_ARM64_BTI
684 : [ilog2(VM_ARM64_BTI)] = "bt",
685 : #endif
686 : #ifdef CONFIG_MEM_SOFT_DIRTY
687 : [ilog2(VM_SOFTDIRTY)] = "sd",
688 : #endif
689 : [ilog2(VM_MIXEDMAP)] = "mm",
690 : [ilog2(VM_HUGEPAGE)] = "hg",
691 : [ilog2(VM_NOHUGEPAGE)] = "nh",
692 : [ilog2(VM_MERGEABLE)] = "mg",
693 : [ilog2(VM_UFFD_MISSING)]= "um",
694 : [ilog2(VM_UFFD_WP)] = "uw",
695 : #ifdef CONFIG_ARM64_MTE
696 : [ilog2(VM_MTE)] = "mt",
697 : [ilog2(VM_MTE_ALLOWED)] = "",
698 : #endif
699 : #ifdef CONFIG_ARCH_HAS_PKEYS
700 : /* These come out via ProtectionKey: */
701 : [ilog2(VM_PKEY_BIT0)] = "",
702 : [ilog2(VM_PKEY_BIT1)] = "",
703 : [ilog2(VM_PKEY_BIT2)] = "",
704 : [ilog2(VM_PKEY_BIT3)] = "",
705 : #if VM_PKEY_BIT4
706 : [ilog2(VM_PKEY_BIT4)] = "",
707 : #endif
708 : #endif /* CONFIG_ARCH_HAS_PKEYS */
709 : #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
710 : [ilog2(VM_UFFD_MINOR)] = "ui",
711 : #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
712 : };
713 : size_t i;
714 :
715 0 : seq_puts(m, "VmFlags: ");
716 0 : for (i = 0; i < BITS_PER_LONG; i++) {
717 0 : if (!mnemonics[i][0])
718 0 : continue;
719 0 : if (vma->vm_flags & (1UL << i)) {
720 0 : seq_putc(m, mnemonics[i][0]);
721 0 : seq_putc(m, mnemonics[i][1]);
722 0 : seq_putc(m, ' ');
723 : }
724 : }
725 0 : seq_putc(m, '\n');
726 0 : }
727 :
728 : #ifdef CONFIG_HUGETLB_PAGE
729 : static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
730 : unsigned long addr, unsigned long end,
731 : struct mm_walk *walk)
732 : {
733 : struct mem_size_stats *mss = walk->private;
734 : struct vm_area_struct *vma = walk->vma;
735 : struct page *page = NULL;
736 : pte_t ptent = ptep_get(pte);
737 :
738 : if (pte_present(ptent)) {
739 : page = vm_normal_page(vma, addr, ptent);
740 : } else if (is_swap_pte(ptent)) {
741 : swp_entry_t swpent = pte_to_swp_entry(ptent);
742 :
743 : if (is_pfn_swap_entry(swpent))
744 : page = pfn_swap_entry_to_page(swpent);
745 : }
746 : if (page) {
747 : if (page_mapcount(page) >= 2 || hugetlb_pmd_shared(pte))
748 : mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
749 : else
750 : mss->private_hugetlb += huge_page_size(hstate_vma(vma));
751 : }
752 : return 0;
753 : }
754 : #else
755 : #define smaps_hugetlb_range NULL
756 : #endif /* HUGETLB_PAGE */
757 :
758 : static const struct mm_walk_ops smaps_walk_ops = {
759 : .pmd_entry = smaps_pte_range,
760 : .hugetlb_entry = smaps_hugetlb_range,
761 : };
762 :
763 : static const struct mm_walk_ops smaps_shmem_walk_ops = {
764 : .pmd_entry = smaps_pte_range,
765 : .hugetlb_entry = smaps_hugetlb_range,
766 : .pte_hole = smaps_pte_hole,
767 : };
768 :
769 : /*
770 : * Gather mem stats from @vma with the indicated beginning
771 : * address @start, and keep them in @mss.
772 : *
773 : * Use vm_start of @vma as the beginning address if @start is 0.
774 : */
775 0 : static void smap_gather_stats(struct vm_area_struct *vma,
776 : struct mem_size_stats *mss, unsigned long start)
777 : {
778 0 : const struct mm_walk_ops *ops = &smaps_walk_ops;
779 :
780 : /* Invalid start */
781 0 : if (start >= vma->vm_end)
782 : return;
783 :
784 0 : if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
785 : /*
786 : * For shared or readonly shmem mappings we know that all
787 : * swapped out pages belong to the shmem object, and we can
788 : * obtain the swap value much more efficiently. For private
789 : * writable mappings, we might have COW pages that are
790 : * not affected by the parent swapped out pages of the shmem
791 : * object, so we have to distinguish them during the page walk.
792 : * Unless we know that the shmem object (or the part mapped by
793 : * our VMA) has no swapped out pages at all.
794 : */
795 0 : unsigned long shmem_swapped = shmem_swap_usage(vma);
796 :
797 0 : if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
798 : !(vma->vm_flags & VM_WRITE))) {
799 0 : mss->swap += shmem_swapped;
800 : } else {
801 : ops = &smaps_shmem_walk_ops;
802 : }
803 : }
804 :
805 : /* mmap_lock is held in m_start */
806 0 : if (!start)
807 0 : walk_page_vma(vma, ops, mss);
808 : else
809 0 : walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
810 : }
811 :
812 : #define SEQ_PUT_DEC(str, val) \
813 : seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
814 :
815 : /* Show the contents common for smaps and smaps_rollup */
816 0 : static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
817 : bool rollup_mode)
818 : {
819 0 : SEQ_PUT_DEC("Rss: ", mss->resident);
820 0 : SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
821 0 : SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT);
822 0 : if (rollup_mode) {
823 : /*
824 : * These are meaningful only for smaps_rollup, otherwise two of
825 : * them are zero, and the other one is the same as Pss.
826 : */
827 0 : SEQ_PUT_DEC(" kB\nPss_Anon: ",
828 : mss->pss_anon >> PSS_SHIFT);
829 0 : SEQ_PUT_DEC(" kB\nPss_File: ",
830 : mss->pss_file >> PSS_SHIFT);
831 0 : SEQ_PUT_DEC(" kB\nPss_Shmem: ",
832 : mss->pss_shmem >> PSS_SHIFT);
833 : }
834 0 : SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
835 0 : SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
836 0 : SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
837 0 : SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
838 0 : SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
839 0 : SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
840 0 : SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
841 0 : SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
842 0 : SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
843 0 : SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp);
844 0 : SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
845 0 : seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
846 0 : mss->private_hugetlb >> 10, 7);
847 0 : SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
848 0 : SEQ_PUT_DEC(" kB\nSwapPss: ",
849 : mss->swap_pss >> PSS_SHIFT);
850 0 : SEQ_PUT_DEC(" kB\nLocked: ",
851 : mss->pss_locked >> PSS_SHIFT);
852 0 : seq_puts(m, " kB\n");
853 0 : }
854 :
855 0 : static int show_smap(struct seq_file *m, void *v)
856 : {
857 0 : struct vm_area_struct *vma = v;
858 : struct mem_size_stats mss;
859 :
860 0 : memset(&mss, 0, sizeof(mss));
861 :
862 0 : smap_gather_stats(vma, &mss, 0);
863 :
864 0 : show_map_vma(m, vma);
865 :
866 0 : SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
867 0 : SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
868 0 : SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
869 0 : seq_puts(m, " kB\n");
870 :
871 0 : __show_smap(m, &mss, false);
872 :
873 0 : seq_printf(m, "THPeligible: %d\n",
874 0 : hugepage_vma_check(vma, vma->vm_flags, true, false, true));
875 :
876 : if (arch_pkeys_enabled())
877 : seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
878 0 : show_smap_vma_flags(m, vma);
879 :
880 0 : return 0;
881 : }
882 :
883 0 : static int show_smaps_rollup(struct seq_file *m, void *v)
884 : {
885 0 : struct proc_maps_private *priv = m->private;
886 : struct mem_size_stats mss;
887 0 : struct mm_struct *mm = priv->mm;
888 : struct vm_area_struct *vma;
889 0 : unsigned long vma_start = 0, last_vma_end = 0;
890 0 : int ret = 0;
891 0 : VMA_ITERATOR(vmi, mm, 0);
892 :
893 0 : priv->task = get_proc_task(priv->inode);
894 0 : if (!priv->task)
895 : return -ESRCH;
896 :
897 0 : if (!mm || !mmget_not_zero(mm)) {
898 : ret = -ESRCH;
899 : goto out_put_task;
900 : }
901 :
902 0 : memset(&mss, 0, sizeof(mss));
903 :
904 0 : ret = mmap_read_lock_killable(mm);
905 0 : if (ret)
906 : goto out_put_mm;
907 :
908 0 : hold_task_mempolicy(priv);
909 0 : vma = vma_next(&vmi);
910 :
911 0 : if (unlikely(!vma))
912 : goto empty_set;
913 :
914 0 : vma_start = vma->vm_start;
915 : do {
916 0 : smap_gather_stats(vma, &mss, 0);
917 0 : last_vma_end = vma->vm_end;
918 :
919 : /*
920 : * Release mmap_lock temporarily if someone wants to
921 : * access it for write request.
922 : */
923 0 : if (mmap_lock_is_contended(mm)) {
924 0 : vma_iter_invalidate(&vmi);
925 0 : mmap_read_unlock(mm);
926 0 : ret = mmap_read_lock_killable(mm);
927 0 : if (ret) {
928 : release_task_mempolicy(priv);
929 : goto out_put_mm;
930 : }
931 :
932 : /*
933 : * After dropping the lock, there are four cases to
934 : * consider. See the following example for explanation.
935 : *
936 : * +------+------+-----------+
937 : * | VMA1 | VMA2 | VMA3 |
938 : * +------+------+-----------+
939 : * | | | |
940 : * 4k 8k 16k 400k
941 : *
942 : * Suppose we drop the lock after reading VMA2 due to
943 : * contention, then we get:
944 : *
945 : * last_vma_end = 16k
946 : *
947 : * 1) VMA2 is freed, but VMA3 exists:
948 : *
949 : * vma_next(vmi) will return VMA3.
950 : * In this case, just continue from VMA3.
951 : *
952 : * 2) VMA2 still exists:
953 : *
954 : * vma_next(vmi) will return VMA3.
955 : * In this case, just continue from VMA3.
956 : *
957 : * 3) No more VMAs can be found:
958 : *
959 : * vma_next(vmi) will return NULL.
960 : * No more things to do, just break.
961 : *
962 : * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
963 : *
964 : * vma_next(vmi) will return VMA' whose range
965 : * contains last_vma_end.
966 : * Iterate VMA' from last_vma_end.
967 : */
968 0 : vma = vma_next(&vmi);
969 : /* Case 3 above */
970 0 : if (!vma)
971 : break;
972 :
973 : /* Case 1 and 2 above */
974 0 : if (vma->vm_start >= last_vma_end)
975 0 : continue;
976 :
977 : /* Case 4 above */
978 0 : if (vma->vm_end > last_vma_end)
979 0 : smap_gather_stats(vma, &mss, last_vma_end);
980 : }
981 0 : } for_each_vma(vmi, vma);
982 :
983 : empty_set:
984 0 : show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0);
985 0 : seq_pad(m, ' ');
986 0 : seq_puts(m, "[rollup]\n");
987 :
988 0 : __show_smap(m, &mss, true);
989 :
990 0 : release_task_mempolicy(priv);
991 : mmap_read_unlock(mm);
992 :
993 : out_put_mm:
994 0 : mmput(mm);
995 : out_put_task:
996 0 : put_task_struct(priv->task);
997 0 : priv->task = NULL;
998 :
999 0 : return ret;
1000 : }
1001 : #undef SEQ_PUT_DEC
1002 :
1003 : static const struct seq_operations proc_pid_smaps_op = {
1004 : .start = m_start,
1005 : .next = m_next,
1006 : .stop = m_stop,
1007 : .show = show_smap
1008 : };
1009 :
1010 0 : static int pid_smaps_open(struct inode *inode, struct file *file)
1011 : {
1012 0 : return do_maps_open(inode, file, &proc_pid_smaps_op);
1013 : }
1014 :
1015 0 : static int smaps_rollup_open(struct inode *inode, struct file *file)
1016 : {
1017 : int ret;
1018 : struct proc_maps_private *priv;
1019 :
1020 0 : priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
1021 0 : if (!priv)
1022 : return -ENOMEM;
1023 :
1024 0 : ret = single_open(file, show_smaps_rollup, priv);
1025 0 : if (ret)
1026 : goto out_free;
1027 :
1028 0 : priv->inode = inode;
1029 0 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
1030 0 : if (IS_ERR(priv->mm)) {
1031 0 : ret = PTR_ERR(priv->mm);
1032 :
1033 0 : single_release(inode, file);
1034 0 : goto out_free;
1035 : }
1036 :
1037 : return 0;
1038 :
1039 : out_free:
1040 0 : kfree(priv);
1041 0 : return ret;
1042 : }
1043 :
1044 0 : static int smaps_rollup_release(struct inode *inode, struct file *file)
1045 : {
1046 0 : struct seq_file *seq = file->private_data;
1047 0 : struct proc_maps_private *priv = seq->private;
1048 :
1049 0 : if (priv->mm)
1050 0 : mmdrop(priv->mm);
1051 :
1052 0 : kfree(priv);
1053 0 : return single_release(inode, file);
1054 : }
1055 :
1056 : const struct file_operations proc_pid_smaps_operations = {
1057 : .open = pid_smaps_open,
1058 : .read = seq_read,
1059 : .llseek = seq_lseek,
1060 : .release = proc_map_release,
1061 : };
1062 :
1063 : const struct file_operations proc_pid_smaps_rollup_operations = {
1064 : .open = smaps_rollup_open,
1065 : .read = seq_read,
1066 : .llseek = seq_lseek,
1067 : .release = smaps_rollup_release,
1068 : };
1069 :
1070 : enum clear_refs_types {
1071 : CLEAR_REFS_ALL = 1,
1072 : CLEAR_REFS_ANON,
1073 : CLEAR_REFS_MAPPED,
1074 : CLEAR_REFS_SOFT_DIRTY,
1075 : CLEAR_REFS_MM_HIWATER_RSS,
1076 : CLEAR_REFS_LAST,
1077 : };
1078 :
1079 : struct clear_refs_private {
1080 : enum clear_refs_types type;
1081 : };
1082 :
1083 : #ifdef CONFIG_MEM_SOFT_DIRTY
1084 :
1085 : static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1086 : {
1087 : struct page *page;
1088 :
1089 : if (!pte_write(pte))
1090 : return false;
1091 : if (!is_cow_mapping(vma->vm_flags))
1092 : return false;
1093 : if (likely(!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)))
1094 : return false;
1095 : page = vm_normal_page(vma, addr, pte);
1096 : if (!page)
1097 : return false;
1098 : return page_maybe_dma_pinned(page);
1099 : }
1100 :
1101 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1102 : unsigned long addr, pte_t *pte)
1103 : {
1104 : /*
1105 : * The soft-dirty tracker uses #PF-s to catch writes
1106 : * to pages, so write-protect the pte as well. See the
1107 : * Documentation/admin-guide/mm/soft-dirty.rst for full description
1108 : * of how soft-dirty works.
1109 : */
1110 : pte_t ptent = ptep_get(pte);
1111 :
1112 : if (pte_present(ptent)) {
1113 : pte_t old_pte;
1114 :
1115 : if (pte_is_pinned(vma, addr, ptent))
1116 : return;
1117 : old_pte = ptep_modify_prot_start(vma, addr, pte);
1118 : ptent = pte_wrprotect(old_pte);
1119 : ptent = pte_clear_soft_dirty(ptent);
1120 : ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
1121 : } else if (is_swap_pte(ptent)) {
1122 : ptent = pte_swp_clear_soft_dirty(ptent);
1123 : set_pte_at(vma->vm_mm, addr, pte, ptent);
1124 : }
1125 : }
1126 : #else
1127 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1128 : unsigned long addr, pte_t *pte)
1129 : {
1130 : }
1131 : #endif
1132 :
1133 : #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1134 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1135 : unsigned long addr, pmd_t *pmdp)
1136 : {
1137 : pmd_t old, pmd = *pmdp;
1138 :
1139 : if (pmd_present(pmd)) {
1140 : /* See comment in change_huge_pmd() */
1141 : old = pmdp_invalidate(vma, addr, pmdp);
1142 : if (pmd_dirty(old))
1143 : pmd = pmd_mkdirty(pmd);
1144 : if (pmd_young(old))
1145 : pmd = pmd_mkyoung(pmd);
1146 :
1147 : pmd = pmd_wrprotect(pmd);
1148 : pmd = pmd_clear_soft_dirty(pmd);
1149 :
1150 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1151 : } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
1152 : pmd = pmd_swp_clear_soft_dirty(pmd);
1153 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1154 : }
1155 : }
1156 : #else
1157 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1158 : unsigned long addr, pmd_t *pmdp)
1159 : {
1160 : }
1161 : #endif
1162 :
1163 0 : static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
1164 : unsigned long end, struct mm_walk *walk)
1165 : {
1166 0 : struct clear_refs_private *cp = walk->private;
1167 0 : struct vm_area_struct *vma = walk->vma;
1168 : pte_t *pte, ptent;
1169 : spinlock_t *ptl;
1170 : struct page *page;
1171 :
1172 0 : ptl = pmd_trans_huge_lock(pmd, vma);
1173 : if (ptl) {
1174 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1175 : clear_soft_dirty_pmd(vma, addr, pmd);
1176 : goto out;
1177 : }
1178 :
1179 : if (!pmd_present(*pmd))
1180 : goto out;
1181 :
1182 : page = pmd_page(*pmd);
1183 :
1184 : /* Clear accessed and referenced bits. */
1185 : pmdp_test_and_clear_young(vma, addr, pmd);
1186 : test_and_clear_page_young(page);
1187 : ClearPageReferenced(page);
1188 : out:
1189 : spin_unlock(ptl);
1190 : return 0;
1191 : }
1192 :
1193 0 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1194 0 : if (!pte) {
1195 0 : walk->action = ACTION_AGAIN;
1196 0 : return 0;
1197 : }
1198 0 : for (; addr != end; pte++, addr += PAGE_SIZE) {
1199 0 : ptent = ptep_get(pte);
1200 :
1201 0 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1202 0 : clear_soft_dirty(vma, addr, pte);
1203 0 : continue;
1204 : }
1205 :
1206 0 : if (!pte_present(ptent))
1207 0 : continue;
1208 :
1209 0 : page = vm_normal_page(vma, addr, ptent);
1210 0 : if (!page)
1211 0 : continue;
1212 :
1213 : /* Clear accessed and referenced bits. */
1214 0 : ptep_test_and_clear_young(vma, addr, pte);
1215 0 : test_and_clear_page_young(page);
1216 : ClearPageReferenced(page);
1217 : }
1218 0 : pte_unmap_unlock(pte - 1, ptl);
1219 0 : cond_resched();
1220 0 : return 0;
1221 : }
1222 :
1223 0 : static int clear_refs_test_walk(unsigned long start, unsigned long end,
1224 : struct mm_walk *walk)
1225 : {
1226 0 : struct clear_refs_private *cp = walk->private;
1227 0 : struct vm_area_struct *vma = walk->vma;
1228 :
1229 0 : if (vma->vm_flags & VM_PFNMAP)
1230 : return 1;
1231 :
1232 : /*
1233 : * Writing 1 to /proc/pid/clear_refs affects all pages.
1234 : * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1235 : * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1236 : * Writing 4 to /proc/pid/clear_refs affects all pages.
1237 : */
1238 0 : if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1239 : return 1;
1240 0 : if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1241 : return 1;
1242 0 : return 0;
1243 : }
1244 :
1245 : static const struct mm_walk_ops clear_refs_walk_ops = {
1246 : .pmd_entry = clear_refs_pte_range,
1247 : .test_walk = clear_refs_test_walk,
1248 : };
1249 :
1250 0 : static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1251 : size_t count, loff_t *ppos)
1252 : {
1253 : struct task_struct *task;
1254 : char buffer[PROC_NUMBUF];
1255 : struct mm_struct *mm;
1256 : struct vm_area_struct *vma;
1257 : enum clear_refs_types type;
1258 : int itype;
1259 : int rv;
1260 :
1261 0 : memset(buffer, 0, sizeof(buffer));
1262 0 : if (count > sizeof(buffer) - 1)
1263 0 : count = sizeof(buffer) - 1;
1264 0 : if (copy_from_user(buffer, buf, count))
1265 : return -EFAULT;
1266 0 : rv = kstrtoint(strstrip(buffer), 10, &itype);
1267 0 : if (rv < 0)
1268 0 : return rv;
1269 0 : type = (enum clear_refs_types)itype;
1270 0 : if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1271 : return -EINVAL;
1272 :
1273 0 : task = get_proc_task(file_inode(file));
1274 0 : if (!task)
1275 : return -ESRCH;
1276 0 : mm = get_task_mm(task);
1277 0 : if (mm) {
1278 0 : VMA_ITERATOR(vmi, mm, 0);
1279 : struct mmu_notifier_range range;
1280 0 : struct clear_refs_private cp = {
1281 : .type = type,
1282 : };
1283 :
1284 0 : if (mmap_write_lock_killable(mm)) {
1285 : count = -EINTR;
1286 : goto out_mm;
1287 : }
1288 0 : if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1289 : /*
1290 : * Writing 5 to /proc/pid/clear_refs resets the peak
1291 : * resident set size to this mm's current rss value.
1292 : */
1293 : reset_mm_hiwater_rss(mm);
1294 : goto out_unlock;
1295 : }
1296 :
1297 0 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1298 0 : for_each_vma(vmi, vma) {
1299 : if (!(vma->vm_flags & VM_SOFTDIRTY))
1300 0 : continue;
1301 : vm_flags_clear(vma, VM_SOFTDIRTY);
1302 : vma_set_page_prot(vma);
1303 : }
1304 :
1305 0 : inc_tlb_flush_pending(mm);
1306 : mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
1307 : 0, mm, 0, -1UL);
1308 : mmu_notifier_invalidate_range_start(&range);
1309 : }
1310 0 : walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp);
1311 0 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1312 0 : mmu_notifier_invalidate_range_end(&range);
1313 0 : flush_tlb_mm(mm);
1314 : dec_tlb_flush_pending(mm);
1315 : }
1316 : out_unlock:
1317 : mmap_write_unlock(mm);
1318 : out_mm:
1319 0 : mmput(mm);
1320 : }
1321 0 : put_task_struct(task);
1322 :
1323 0 : return count;
1324 : }
1325 :
1326 : const struct file_operations proc_clear_refs_operations = {
1327 : .write = clear_refs_write,
1328 : .llseek = noop_llseek,
1329 : };
1330 :
1331 : typedef struct {
1332 : u64 pme;
1333 : } pagemap_entry_t;
1334 :
1335 : struct pagemapread {
1336 : int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
1337 : pagemap_entry_t *buffer;
1338 : bool show_pfn;
1339 : };
1340 :
1341 : #define PAGEMAP_WALK_SIZE (PMD_SIZE)
1342 : #define PAGEMAP_WALK_MASK (PMD_MASK)
1343 :
1344 : #define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1345 : #define PM_PFRAME_BITS 55
1346 : #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1347 : #define PM_SOFT_DIRTY BIT_ULL(55)
1348 : #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1349 : #define PM_UFFD_WP BIT_ULL(57)
1350 : #define PM_FILE BIT_ULL(61)
1351 : #define PM_SWAP BIT_ULL(62)
1352 : #define PM_PRESENT BIT_ULL(63)
1353 :
1354 : #define PM_END_OF_BUFFER 1
1355 :
1356 : static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1357 : {
1358 0 : return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1359 : }
1360 :
1361 : static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1362 : struct pagemapread *pm)
1363 : {
1364 0 : pm->buffer[pm->pos++] = *pme;
1365 0 : if (pm->pos >= pm->len)
1366 : return PM_END_OF_BUFFER;
1367 : return 0;
1368 : }
1369 :
1370 0 : static int pagemap_pte_hole(unsigned long start, unsigned long end,
1371 : __always_unused int depth, struct mm_walk *walk)
1372 : {
1373 0 : struct pagemapread *pm = walk->private;
1374 0 : unsigned long addr = start;
1375 0 : int err = 0;
1376 :
1377 0 : while (addr < end) {
1378 0 : struct vm_area_struct *vma = find_vma(walk->mm, addr);
1379 0 : pagemap_entry_t pme = make_pme(0, 0);
1380 : /* End of address space hole, which we mark as non-present. */
1381 : unsigned long hole_end;
1382 :
1383 0 : if (vma)
1384 0 : hole_end = min(end, vma->vm_start);
1385 : else
1386 : hole_end = end;
1387 :
1388 0 : for (; addr < hole_end; addr += PAGE_SIZE) {
1389 0 : err = add_to_pagemap(addr, &pme, pm);
1390 0 : if (err)
1391 : goto out;
1392 : }
1393 :
1394 0 : if (!vma)
1395 : break;
1396 :
1397 : /* Addresses in the VMA. */
1398 : if (vma->vm_flags & VM_SOFTDIRTY)
1399 : pme = make_pme(0, PM_SOFT_DIRTY);
1400 0 : for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1401 0 : err = add_to_pagemap(addr, &pme, pm);
1402 0 : if (err)
1403 : goto out;
1404 : }
1405 : }
1406 : out:
1407 0 : return err;
1408 : }
1409 :
1410 0 : static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1411 : struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1412 : {
1413 0 : u64 frame = 0, flags = 0;
1414 0 : struct page *page = NULL;
1415 0 : bool migration = false;
1416 :
1417 0 : if (pte_present(pte)) {
1418 0 : if (pm->show_pfn)
1419 0 : frame = pte_pfn(pte);
1420 0 : flags |= PM_PRESENT;
1421 0 : page = vm_normal_page(vma, addr, pte);
1422 0 : if (pte_soft_dirty(pte))
1423 : flags |= PM_SOFT_DIRTY;
1424 : if (pte_uffd_wp(pte))
1425 : flags |= PM_UFFD_WP;
1426 0 : } else if (is_swap_pte(pte)) {
1427 : swp_entry_t entry;
1428 0 : if (pte_swp_soft_dirty(pte))
1429 : flags |= PM_SOFT_DIRTY;
1430 : if (pte_swp_uffd_wp(pte))
1431 : flags |= PM_UFFD_WP;
1432 0 : entry = pte_to_swp_entry(pte);
1433 0 : if (pm->show_pfn) {
1434 : pgoff_t offset;
1435 : /*
1436 : * For PFN swap offsets, keeping the offset field
1437 : * to be PFN only to be compatible with old smaps.
1438 : */
1439 0 : if (is_pfn_swap_entry(entry))
1440 0 : offset = swp_offset_pfn(entry);
1441 : else
1442 0 : offset = swp_offset(entry);
1443 0 : frame = swp_type(entry) |
1444 0 : (offset << MAX_SWAPFILES_SHIFT);
1445 : }
1446 0 : flags |= PM_SWAP;
1447 0 : migration = is_migration_entry(entry);
1448 0 : if (is_pfn_swap_entry(entry))
1449 0 : page = pfn_swap_entry_to_page(entry);
1450 : if (pte_marker_entry_uffd_wp(entry))
1451 : flags |= PM_UFFD_WP;
1452 : }
1453 :
1454 0 : if (page && !PageAnon(page))
1455 0 : flags |= PM_FILE;
1456 0 : if (page && !migration && page_mapcount(page) == 1)
1457 0 : flags |= PM_MMAP_EXCLUSIVE;
1458 : if (vma->vm_flags & VM_SOFTDIRTY)
1459 : flags |= PM_SOFT_DIRTY;
1460 :
1461 0 : return make_pme(frame, flags);
1462 : }
1463 :
1464 0 : static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1465 : struct mm_walk *walk)
1466 : {
1467 0 : struct vm_area_struct *vma = walk->vma;
1468 0 : struct pagemapread *pm = walk->private;
1469 : spinlock_t *ptl;
1470 : pte_t *pte, *orig_pte;
1471 0 : int err = 0;
1472 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1473 : bool migration = false;
1474 :
1475 : ptl = pmd_trans_huge_lock(pmdp, vma);
1476 : if (ptl) {
1477 : u64 flags = 0, frame = 0;
1478 : pmd_t pmd = *pmdp;
1479 : struct page *page = NULL;
1480 :
1481 : if (vma->vm_flags & VM_SOFTDIRTY)
1482 : flags |= PM_SOFT_DIRTY;
1483 :
1484 : if (pmd_present(pmd)) {
1485 : page = pmd_page(pmd);
1486 :
1487 : flags |= PM_PRESENT;
1488 : if (pmd_soft_dirty(pmd))
1489 : flags |= PM_SOFT_DIRTY;
1490 : if (pmd_uffd_wp(pmd))
1491 : flags |= PM_UFFD_WP;
1492 : if (pm->show_pfn)
1493 : frame = pmd_pfn(pmd) +
1494 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1495 : }
1496 : #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1497 : else if (is_swap_pmd(pmd)) {
1498 : swp_entry_t entry = pmd_to_swp_entry(pmd);
1499 : unsigned long offset;
1500 :
1501 : if (pm->show_pfn) {
1502 : if (is_pfn_swap_entry(entry))
1503 : offset = swp_offset_pfn(entry);
1504 : else
1505 : offset = swp_offset(entry);
1506 : offset = offset +
1507 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1508 : frame = swp_type(entry) |
1509 : (offset << MAX_SWAPFILES_SHIFT);
1510 : }
1511 : flags |= PM_SWAP;
1512 : if (pmd_swp_soft_dirty(pmd))
1513 : flags |= PM_SOFT_DIRTY;
1514 : if (pmd_swp_uffd_wp(pmd))
1515 : flags |= PM_UFFD_WP;
1516 : VM_BUG_ON(!is_pmd_migration_entry(pmd));
1517 : migration = is_migration_entry(entry);
1518 : page = pfn_swap_entry_to_page(entry);
1519 : }
1520 : #endif
1521 :
1522 : if (page && !migration && page_mapcount(page) == 1)
1523 : flags |= PM_MMAP_EXCLUSIVE;
1524 :
1525 : for (; addr != end; addr += PAGE_SIZE) {
1526 : pagemap_entry_t pme = make_pme(frame, flags);
1527 :
1528 : err = add_to_pagemap(addr, &pme, pm);
1529 : if (err)
1530 : break;
1531 : if (pm->show_pfn) {
1532 : if (flags & PM_PRESENT)
1533 : frame++;
1534 : else if (flags & PM_SWAP)
1535 : frame += (1 << MAX_SWAPFILES_SHIFT);
1536 : }
1537 : }
1538 : spin_unlock(ptl);
1539 : return err;
1540 : }
1541 : #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1542 :
1543 : /*
1544 : * We can assume that @vma always points to a valid one and @end never
1545 : * goes beyond vma->vm_end.
1546 : */
1547 0 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1548 0 : if (!pte) {
1549 0 : walk->action = ACTION_AGAIN;
1550 0 : return err;
1551 : }
1552 0 : for (; addr < end; pte++, addr += PAGE_SIZE) {
1553 : pagemap_entry_t pme;
1554 :
1555 0 : pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte));
1556 0 : err = add_to_pagemap(addr, &pme, pm);
1557 0 : if (err)
1558 : break;
1559 : }
1560 0 : pte_unmap_unlock(orig_pte, ptl);
1561 :
1562 0 : cond_resched();
1563 :
1564 0 : return err;
1565 : }
1566 :
1567 : #ifdef CONFIG_HUGETLB_PAGE
1568 : /* This function walks within one hugetlb entry in the single call */
1569 : static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1570 : unsigned long addr, unsigned long end,
1571 : struct mm_walk *walk)
1572 : {
1573 : struct pagemapread *pm = walk->private;
1574 : struct vm_area_struct *vma = walk->vma;
1575 : u64 flags = 0, frame = 0;
1576 : int err = 0;
1577 : pte_t pte;
1578 :
1579 : if (vma->vm_flags & VM_SOFTDIRTY)
1580 : flags |= PM_SOFT_DIRTY;
1581 :
1582 : pte = huge_ptep_get(ptep);
1583 : if (pte_present(pte)) {
1584 : struct page *page = pte_page(pte);
1585 :
1586 : if (!PageAnon(page))
1587 : flags |= PM_FILE;
1588 :
1589 : if (page_mapcount(page) == 1)
1590 : flags |= PM_MMAP_EXCLUSIVE;
1591 :
1592 : if (huge_pte_uffd_wp(pte))
1593 : flags |= PM_UFFD_WP;
1594 :
1595 : flags |= PM_PRESENT;
1596 : if (pm->show_pfn)
1597 : frame = pte_pfn(pte) +
1598 : ((addr & ~hmask) >> PAGE_SHIFT);
1599 : } else if (pte_swp_uffd_wp_any(pte)) {
1600 : flags |= PM_UFFD_WP;
1601 : }
1602 :
1603 : for (; addr != end; addr += PAGE_SIZE) {
1604 : pagemap_entry_t pme = make_pme(frame, flags);
1605 :
1606 : err = add_to_pagemap(addr, &pme, pm);
1607 : if (err)
1608 : return err;
1609 : if (pm->show_pfn && (flags & PM_PRESENT))
1610 : frame++;
1611 : }
1612 :
1613 : cond_resched();
1614 :
1615 : return err;
1616 : }
1617 : #else
1618 : #define pagemap_hugetlb_range NULL
1619 : #endif /* HUGETLB_PAGE */
1620 :
1621 : static const struct mm_walk_ops pagemap_ops = {
1622 : .pmd_entry = pagemap_pmd_range,
1623 : .pte_hole = pagemap_pte_hole,
1624 : .hugetlb_entry = pagemap_hugetlb_range,
1625 : };
1626 :
1627 : /*
1628 : * /proc/pid/pagemap - an array mapping virtual pages to pfns
1629 : *
1630 : * For each page in the address space, this file contains one 64-bit entry
1631 : * consisting of the following:
1632 : *
1633 : * Bits 0-54 page frame number (PFN) if present
1634 : * Bits 0-4 swap type if swapped
1635 : * Bits 5-54 swap offset if swapped
1636 : * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1637 : * Bit 56 page exclusively mapped
1638 : * Bit 57 pte is uffd-wp write-protected
1639 : * Bits 58-60 zero
1640 : * Bit 61 page is file-page or shared-anon
1641 : * Bit 62 page swapped
1642 : * Bit 63 page present
1643 : *
1644 : * If the page is not present but in swap, then the PFN contains an
1645 : * encoding of the swap file number and the page's offset into the
1646 : * swap. Unmapped pages return a null PFN. This allows determining
1647 : * precisely which pages are mapped (or in swap) and comparing mapped
1648 : * pages between processes.
1649 : *
1650 : * Efficient users of this interface will use /proc/pid/maps to
1651 : * determine which areas of memory are actually mapped and llseek to
1652 : * skip over unmapped regions.
1653 : */
1654 0 : static ssize_t pagemap_read(struct file *file, char __user *buf,
1655 : size_t count, loff_t *ppos)
1656 : {
1657 0 : struct mm_struct *mm = file->private_data;
1658 : struct pagemapread pm;
1659 : unsigned long src;
1660 : unsigned long svpfn;
1661 : unsigned long start_vaddr;
1662 : unsigned long end_vaddr;
1663 0 : int ret = 0, copied = 0;
1664 :
1665 0 : if (!mm || !mmget_not_zero(mm))
1666 : goto out;
1667 :
1668 0 : ret = -EINVAL;
1669 : /* file position must be aligned */
1670 0 : if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1671 : goto out_mm;
1672 :
1673 0 : ret = 0;
1674 0 : if (!count)
1675 : goto out_mm;
1676 :
1677 : /* do not disclose physical addresses: attack vector */
1678 0 : pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1679 :
1680 0 : pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1681 0 : pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1682 0 : ret = -ENOMEM;
1683 0 : if (!pm.buffer)
1684 : goto out_mm;
1685 :
1686 0 : src = *ppos;
1687 0 : svpfn = src / PM_ENTRY_BYTES;
1688 0 : end_vaddr = mm->task_size;
1689 :
1690 : /* watch out for wraparound */
1691 0 : start_vaddr = end_vaddr;
1692 0 : if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) {
1693 : unsigned long end;
1694 :
1695 0 : ret = mmap_read_lock_killable(mm);
1696 0 : if (ret)
1697 : goto out_free;
1698 0 : start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT);
1699 0 : mmap_read_unlock(mm);
1700 :
1701 0 : end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT);
1702 0 : if (end >= start_vaddr && end < mm->task_size)
1703 0 : end_vaddr = end;
1704 : }
1705 :
1706 : /* Ensure the address is inside the task */
1707 0 : if (start_vaddr > mm->task_size)
1708 0 : start_vaddr = end_vaddr;
1709 :
1710 : ret = 0;
1711 0 : while (count && (start_vaddr < end_vaddr)) {
1712 : int len;
1713 : unsigned long end;
1714 :
1715 0 : pm.pos = 0;
1716 0 : end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1717 : /* overflow ? */
1718 0 : if (end < start_vaddr || end > end_vaddr)
1719 0 : end = end_vaddr;
1720 0 : ret = mmap_read_lock_killable(mm);
1721 0 : if (ret)
1722 : goto out_free;
1723 0 : ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm);
1724 0 : mmap_read_unlock(mm);
1725 0 : start_vaddr = end;
1726 :
1727 0 : len = min(count, PM_ENTRY_BYTES * pm.pos);
1728 0 : if (copy_to_user(buf, pm.buffer, len)) {
1729 : ret = -EFAULT;
1730 : goto out_free;
1731 : }
1732 0 : copied += len;
1733 0 : buf += len;
1734 0 : count -= len;
1735 : }
1736 0 : *ppos += copied;
1737 0 : if (!ret || ret == PM_END_OF_BUFFER)
1738 0 : ret = copied;
1739 :
1740 : out_free:
1741 0 : kfree(pm.buffer);
1742 : out_mm:
1743 0 : mmput(mm);
1744 : out:
1745 0 : return ret;
1746 : }
1747 :
1748 0 : static int pagemap_open(struct inode *inode, struct file *file)
1749 : {
1750 : struct mm_struct *mm;
1751 :
1752 0 : mm = proc_mem_open(inode, PTRACE_MODE_READ);
1753 0 : if (IS_ERR(mm))
1754 0 : return PTR_ERR(mm);
1755 0 : file->private_data = mm;
1756 0 : return 0;
1757 : }
1758 :
1759 0 : static int pagemap_release(struct inode *inode, struct file *file)
1760 : {
1761 0 : struct mm_struct *mm = file->private_data;
1762 :
1763 0 : if (mm)
1764 : mmdrop(mm);
1765 0 : return 0;
1766 : }
1767 :
1768 : const struct file_operations proc_pagemap_operations = {
1769 : .llseek = mem_lseek, /* borrow this */
1770 : .read = pagemap_read,
1771 : .open = pagemap_open,
1772 : .release = pagemap_release,
1773 : };
1774 : #endif /* CONFIG_PROC_PAGE_MONITOR */
1775 :
1776 : #ifdef CONFIG_NUMA
1777 :
1778 : struct numa_maps {
1779 : unsigned long pages;
1780 : unsigned long anon;
1781 : unsigned long active;
1782 : unsigned long writeback;
1783 : unsigned long mapcount_max;
1784 : unsigned long dirty;
1785 : unsigned long swapcache;
1786 : unsigned long node[MAX_NUMNODES];
1787 : };
1788 :
1789 : struct numa_maps_private {
1790 : struct proc_maps_private proc_maps;
1791 : struct numa_maps md;
1792 : };
1793 :
1794 : static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1795 : unsigned long nr_pages)
1796 : {
1797 : int count = page_mapcount(page);
1798 :
1799 : md->pages += nr_pages;
1800 : if (pte_dirty || PageDirty(page))
1801 : md->dirty += nr_pages;
1802 :
1803 : if (PageSwapCache(page))
1804 : md->swapcache += nr_pages;
1805 :
1806 : if (PageActive(page) || PageUnevictable(page))
1807 : md->active += nr_pages;
1808 :
1809 : if (PageWriteback(page))
1810 : md->writeback += nr_pages;
1811 :
1812 : if (PageAnon(page))
1813 : md->anon += nr_pages;
1814 :
1815 : if (count > md->mapcount_max)
1816 : md->mapcount_max = count;
1817 :
1818 : md->node[page_to_nid(page)] += nr_pages;
1819 : }
1820 :
1821 : static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1822 : unsigned long addr)
1823 : {
1824 : struct page *page;
1825 : int nid;
1826 :
1827 : if (!pte_present(pte))
1828 : return NULL;
1829 :
1830 : page = vm_normal_page(vma, addr, pte);
1831 : if (!page || is_zone_device_page(page))
1832 : return NULL;
1833 :
1834 : if (PageReserved(page))
1835 : return NULL;
1836 :
1837 : nid = page_to_nid(page);
1838 : if (!node_isset(nid, node_states[N_MEMORY]))
1839 : return NULL;
1840 :
1841 : return page;
1842 : }
1843 :
1844 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1845 : static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1846 : struct vm_area_struct *vma,
1847 : unsigned long addr)
1848 : {
1849 : struct page *page;
1850 : int nid;
1851 :
1852 : if (!pmd_present(pmd))
1853 : return NULL;
1854 :
1855 : page = vm_normal_page_pmd(vma, addr, pmd);
1856 : if (!page)
1857 : return NULL;
1858 :
1859 : if (PageReserved(page))
1860 : return NULL;
1861 :
1862 : nid = page_to_nid(page);
1863 : if (!node_isset(nid, node_states[N_MEMORY]))
1864 : return NULL;
1865 :
1866 : return page;
1867 : }
1868 : #endif
1869 :
1870 : static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1871 : unsigned long end, struct mm_walk *walk)
1872 : {
1873 : struct numa_maps *md = walk->private;
1874 : struct vm_area_struct *vma = walk->vma;
1875 : spinlock_t *ptl;
1876 : pte_t *orig_pte;
1877 : pte_t *pte;
1878 :
1879 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1880 : ptl = pmd_trans_huge_lock(pmd, vma);
1881 : if (ptl) {
1882 : struct page *page;
1883 :
1884 : page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1885 : if (page)
1886 : gather_stats(page, md, pmd_dirty(*pmd),
1887 : HPAGE_PMD_SIZE/PAGE_SIZE);
1888 : spin_unlock(ptl);
1889 : return 0;
1890 : }
1891 : #endif
1892 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1893 : if (!pte) {
1894 : walk->action = ACTION_AGAIN;
1895 : return 0;
1896 : }
1897 : do {
1898 : pte_t ptent = ptep_get(pte);
1899 : struct page *page = can_gather_numa_stats(ptent, vma, addr);
1900 : if (!page)
1901 : continue;
1902 : gather_stats(page, md, pte_dirty(ptent), 1);
1903 :
1904 : } while (pte++, addr += PAGE_SIZE, addr != end);
1905 : pte_unmap_unlock(orig_pte, ptl);
1906 : cond_resched();
1907 : return 0;
1908 : }
1909 : #ifdef CONFIG_HUGETLB_PAGE
1910 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1911 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1912 : {
1913 : pte_t huge_pte = huge_ptep_get(pte);
1914 : struct numa_maps *md;
1915 : struct page *page;
1916 :
1917 : if (!pte_present(huge_pte))
1918 : return 0;
1919 :
1920 : page = pte_page(huge_pte);
1921 :
1922 : md = walk->private;
1923 : gather_stats(page, md, pte_dirty(huge_pte), 1);
1924 : return 0;
1925 : }
1926 :
1927 : #else
1928 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1929 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1930 : {
1931 : return 0;
1932 : }
1933 : #endif
1934 :
1935 : static const struct mm_walk_ops show_numa_ops = {
1936 : .hugetlb_entry = gather_hugetlb_stats,
1937 : .pmd_entry = gather_pte_stats,
1938 : };
1939 :
1940 : /*
1941 : * Display pages allocated per node and memory policy via /proc.
1942 : */
1943 : static int show_numa_map(struct seq_file *m, void *v)
1944 : {
1945 : struct numa_maps_private *numa_priv = m->private;
1946 : struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1947 : struct vm_area_struct *vma = v;
1948 : struct numa_maps *md = &numa_priv->md;
1949 : struct file *file = vma->vm_file;
1950 : struct mm_struct *mm = vma->vm_mm;
1951 : struct mempolicy *pol;
1952 : char buffer[64];
1953 : int nid;
1954 :
1955 : if (!mm)
1956 : return 0;
1957 :
1958 : /* Ensure we start with an empty set of numa_maps statistics. */
1959 : memset(md, 0, sizeof(*md));
1960 :
1961 : pol = __get_vma_policy(vma, vma->vm_start);
1962 : if (pol) {
1963 : mpol_to_str(buffer, sizeof(buffer), pol);
1964 : mpol_cond_put(pol);
1965 : } else {
1966 : mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1967 : }
1968 :
1969 : seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1970 :
1971 : if (file) {
1972 : seq_puts(m, " file=");
1973 : seq_file_path(m, file, "\n\t= ");
1974 : } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1975 : seq_puts(m, " heap");
1976 : } else if (is_stack(vma)) {
1977 : seq_puts(m, " stack");
1978 : }
1979 :
1980 : if (is_vm_hugetlb_page(vma))
1981 : seq_puts(m, " huge");
1982 :
1983 : /* mmap_lock is held by m_start */
1984 : walk_page_vma(vma, &show_numa_ops, md);
1985 :
1986 : if (!md->pages)
1987 : goto out;
1988 :
1989 : if (md->anon)
1990 : seq_printf(m, " anon=%lu", md->anon);
1991 :
1992 : if (md->dirty)
1993 : seq_printf(m, " dirty=%lu", md->dirty);
1994 :
1995 : if (md->pages != md->anon && md->pages != md->dirty)
1996 : seq_printf(m, " mapped=%lu", md->pages);
1997 :
1998 : if (md->mapcount_max > 1)
1999 : seq_printf(m, " mapmax=%lu", md->mapcount_max);
2000 :
2001 : if (md->swapcache)
2002 : seq_printf(m, " swapcache=%lu", md->swapcache);
2003 :
2004 : if (md->active < md->pages && !is_vm_hugetlb_page(vma))
2005 : seq_printf(m, " active=%lu", md->active);
2006 :
2007 : if (md->writeback)
2008 : seq_printf(m, " writeback=%lu", md->writeback);
2009 :
2010 : for_each_node_state(nid, N_MEMORY)
2011 : if (md->node[nid])
2012 : seq_printf(m, " N%d=%lu", nid, md->node[nid]);
2013 :
2014 : seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
2015 : out:
2016 : seq_putc(m, '\n');
2017 : return 0;
2018 : }
2019 :
2020 : static const struct seq_operations proc_pid_numa_maps_op = {
2021 : .start = m_start,
2022 : .next = m_next,
2023 : .stop = m_stop,
2024 : .show = show_numa_map,
2025 : };
2026 :
2027 : static int pid_numa_maps_open(struct inode *inode, struct file *file)
2028 : {
2029 : return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
2030 : sizeof(struct numa_maps_private));
2031 : }
2032 :
2033 : const struct file_operations proc_pid_numa_maps_operations = {
2034 : .open = pid_numa_maps_open,
2035 : .read = seq_read,
2036 : .llseek = seq_lseek,
2037 : .release = proc_map_release,
2038 : };
2039 :
2040 : #endif /* CONFIG_NUMA */
|
