Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : #include <linux/pagewalk.h>
3 : #include <linux/mm_inline.h>
4 : #include <linux/hugetlb.h>
5 : #include <linux/huge_mm.h>
6 : #include <linux/mount.h>
7 : #include <linux/seq_file.h>
8 : #include <linux/highmem.h>
9 : #include <linux/ptrace.h>
10 : #include <linux/slab.h>
11 : #include <linux/pagemap.h>
12 : #include <linux/mempolicy.h>
13 : #include <linux/rmap.h>
14 : #include <linux/swap.h>
15 : #include <linux/sched/mm.h>
16 : #include <linux/swapops.h>
17 : #include <linux/mmu_notifier.h>
18 : #include <linux/page_idle.h>
19 : #include <linux/shmem_fs.h>
20 : #include <linux/uaccess.h>
21 : #include <linux/pkeys.h>
22 :
23 : #include <asm/elf.h>
24 : #include <asm/tlb.h>
25 : #include <asm/tlbflush.h>
26 : #include "internal.h"
27 :
28 : #define SEQ_PUT_DEC(str, val) \
29 : seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
30 0 : void task_mem(struct seq_file *m, struct mm_struct *mm)
31 : {
32 : unsigned long text, lib, swap, anon, file, shmem;
33 : unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
34 :
35 0 : anon = get_mm_counter(mm, MM_ANONPAGES);
36 0 : file = get_mm_counter(mm, MM_FILEPAGES);
37 0 : shmem = get_mm_counter(mm, MM_SHMEMPAGES);
38 :
39 : /*
40 : * Note: to minimize their overhead, mm maintains hiwater_vm and
41 : * hiwater_rss only when about to *lower* total_vm or rss. Any
42 : * collector of these hiwater stats must therefore get total_vm
43 : * and rss too, which will usually be the higher. Barriers? not
44 : * worth the effort, such snapshots can always be inconsistent.
45 : */
46 0 : hiwater_vm = total_vm = mm->total_vm;
47 0 : if (hiwater_vm < mm->hiwater_vm)
48 0 : hiwater_vm = mm->hiwater_vm;
49 0 : hiwater_rss = total_rss = anon + file + shmem;
50 0 : if (hiwater_rss < mm->hiwater_rss)
51 0 : hiwater_rss = mm->hiwater_rss;
52 :
53 : /* split executable areas between text and lib */
54 0 : text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
55 0 : text = min(text, mm->exec_vm << PAGE_SHIFT);
56 0 : lib = (mm->exec_vm << PAGE_SHIFT) - text;
57 :
58 0 : swap = get_mm_counter(mm, MM_SWAPENTS);
59 0 : SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
60 0 : SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
61 0 : SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
62 0 : SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
63 0 : SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
64 0 : SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
65 0 : SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
66 0 : SEQ_PUT_DEC(" kB\nRssFile:\t", file);
67 0 : SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
68 0 : SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
69 0 : SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
70 0 : seq_put_decimal_ull_width(m,
71 0 : " kB\nVmExe:\t", text >> 10, 8);
72 0 : seq_put_decimal_ull_width(m,
73 0 : " kB\nVmLib:\t", lib >> 10, 8);
74 0 : seq_put_decimal_ull_width(m,
75 0 : " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
76 0 : SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
77 0 : seq_puts(m, " kB\n");
78 0 : hugetlb_report_usage(m, mm);
79 0 : }
80 : #undef SEQ_PUT_DEC
81 :
82 0 : unsigned long task_vsize(struct mm_struct *mm)
83 : {
84 0 : return PAGE_SIZE * mm->total_vm;
85 : }
86 :
87 0 : unsigned long task_statm(struct mm_struct *mm,
88 : unsigned long *shared, unsigned long *text,
89 : unsigned long *data, unsigned long *resident)
90 : {
91 0 : *shared = get_mm_counter(mm, MM_FILEPAGES) +
92 0 : get_mm_counter(mm, MM_SHMEMPAGES);
93 0 : *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
94 0 : >> PAGE_SHIFT;
95 0 : *data = mm->data_vm + mm->stack_vm;
96 0 : *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
97 0 : return mm->total_vm;
98 : }
99 :
100 : #ifdef CONFIG_NUMA
101 : /*
102 : * Save get_task_policy() for show_numa_map().
103 : */
104 : static void hold_task_mempolicy(struct proc_maps_private *priv)
105 : {
106 : struct task_struct *task = priv->task;
107 :
108 : task_lock(task);
109 : priv->task_mempolicy = get_task_policy(task);
110 : mpol_get(priv->task_mempolicy);
111 : task_unlock(task);
112 : }
113 : static void release_task_mempolicy(struct proc_maps_private *priv)
114 : {
115 : mpol_put(priv->task_mempolicy);
116 : }
117 : #else
118 : static void hold_task_mempolicy(struct proc_maps_private *priv)
119 : {
120 : }
121 : static void release_task_mempolicy(struct proc_maps_private *priv)
122 : {
123 : }
124 : #endif
125 :
126 : static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv,
127 : loff_t *ppos)
128 : {
129 0 : struct vm_area_struct *vma = vma_next(&priv->iter);
130 :
131 0 : if (vma) {
132 0 : *ppos = vma->vm_start;
133 : } else {
134 0 : *ppos = -2UL;
135 0 : vma = get_gate_vma(priv->mm);
136 : }
137 :
138 : return vma;
139 : }
140 :
141 0 : static void *m_start(struct seq_file *m, loff_t *ppos)
142 : {
143 0 : struct proc_maps_private *priv = m->private;
144 0 : unsigned long last_addr = *ppos;
145 : struct mm_struct *mm;
146 :
147 : /* See m_next(). Zero at the start or after lseek. */
148 0 : if (last_addr == -1UL)
149 : return NULL;
150 :
151 0 : priv->task = get_proc_task(priv->inode);
152 0 : if (!priv->task)
153 : return ERR_PTR(-ESRCH);
154 :
155 0 : mm = priv->mm;
156 0 : if (!mm || !mmget_not_zero(mm)) {
157 0 : put_task_struct(priv->task);
158 0 : priv->task = NULL;
159 0 : return NULL;
160 : }
161 :
162 0 : if (mmap_read_lock_killable(mm)) {
163 0 : mmput(mm);
164 0 : put_task_struct(priv->task);
165 0 : priv->task = NULL;
166 0 : return ERR_PTR(-EINTR);
167 : }
168 :
169 0 : vma_iter_init(&priv->iter, mm, last_addr);
170 0 : hold_task_mempolicy(priv);
171 0 : if (last_addr == -2UL)
172 : return get_gate_vma(mm);
173 :
174 : return proc_get_vma(priv, ppos);
175 : }
176 :
177 0 : static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
178 : {
179 0 : if (*ppos == -2UL) {
180 0 : *ppos = -1UL;
181 0 : return NULL;
182 : }
183 0 : return proc_get_vma(m->private, ppos);
184 : }
185 :
186 0 : static void m_stop(struct seq_file *m, void *v)
187 : {
188 0 : struct proc_maps_private *priv = m->private;
189 0 : struct mm_struct *mm = priv->mm;
190 :
191 0 : if (!priv->task)
192 : return;
193 :
194 0 : release_task_mempolicy(priv);
195 0 : mmap_read_unlock(mm);
196 0 : mmput(mm);
197 0 : put_task_struct(priv->task);
198 0 : priv->task = NULL;
199 : }
200 :
201 0 : static int proc_maps_open(struct inode *inode, struct file *file,
202 : const struct seq_operations *ops, int psize)
203 : {
204 0 : struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
205 :
206 0 : if (!priv)
207 : return -ENOMEM;
208 :
209 0 : priv->inode = inode;
210 0 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
211 0 : if (IS_ERR(priv->mm)) {
212 0 : int err = PTR_ERR(priv->mm);
213 :
214 0 : seq_release_private(inode, file);
215 0 : return err;
216 : }
217 :
218 : return 0;
219 : }
220 :
221 0 : static int proc_map_release(struct inode *inode, struct file *file)
222 : {
223 0 : struct seq_file *seq = file->private_data;
224 0 : struct proc_maps_private *priv = seq->private;
225 :
226 0 : if (priv->mm)
227 0 : mmdrop(priv->mm);
228 :
229 0 : return seq_release_private(inode, file);
230 : }
231 :
232 : static int do_maps_open(struct inode *inode, struct file *file,
233 : const struct seq_operations *ops)
234 : {
235 0 : return proc_maps_open(inode, file, ops,
236 : sizeof(struct proc_maps_private));
237 : }
238 :
239 : /*
240 : * Indicate if the VMA is a stack for the given task; for
241 : * /proc/PID/maps that is the stack of the main task.
242 : */
243 : static int is_stack(struct vm_area_struct *vma)
244 : {
245 : /*
246 : * We make no effort to guess what a given thread considers to be
247 : * its "stack". It's not even well-defined for programs written
248 : * languages like Go.
249 : */
250 0 : return vma->vm_start <= vma->vm_mm->start_stack &&
251 0 : vma->vm_end >= vma->vm_mm->start_stack;
252 : }
253 :
254 0 : static void show_vma_header_prefix(struct seq_file *m,
255 : unsigned long start, unsigned long end,
256 : vm_flags_t flags, unsigned long long pgoff,
257 : dev_t dev, unsigned long ino)
258 : {
259 0 : seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
260 0 : seq_put_hex_ll(m, NULL, start, 8);
261 0 : seq_put_hex_ll(m, "-", end, 8);
262 0 : seq_putc(m, ' ');
263 0 : seq_putc(m, flags & VM_READ ? 'r' : '-');
264 0 : seq_putc(m, flags & VM_WRITE ? 'w' : '-');
265 0 : seq_putc(m, flags & VM_EXEC ? 'x' : '-');
266 0 : seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
267 0 : seq_put_hex_ll(m, " ", pgoff, 8);
268 0 : seq_put_hex_ll(m, " ", MAJOR(dev), 2);
269 0 : seq_put_hex_ll(m, ":", MINOR(dev), 2);
270 0 : seq_put_decimal_ull(m, " ", ino);
271 0 : seq_putc(m, ' ');
272 0 : }
273 :
274 : static void
275 0 : show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
276 : {
277 0 : struct anon_vma_name *anon_name = NULL;
278 0 : struct mm_struct *mm = vma->vm_mm;
279 0 : struct file *file = vma->vm_file;
280 0 : vm_flags_t flags = vma->vm_flags;
281 0 : unsigned long ino = 0;
282 0 : unsigned long long pgoff = 0;
283 : unsigned long start, end;
284 0 : dev_t dev = 0;
285 0 : const char *name = NULL;
286 :
287 0 : if (file) {
288 0 : struct inode *inode = file_inode(vma->vm_file);
289 0 : dev = inode->i_sb->s_dev;
290 0 : ino = inode->i_ino;
291 0 : pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
292 : }
293 :
294 0 : start = vma->vm_start;
295 0 : end = vma->vm_end;
296 0 : show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
297 : if (mm)
298 : anon_name = anon_vma_name(vma);
299 :
300 : /*
301 : * Print the dentry name for named mappings, and a
302 : * special [heap] marker for the heap:
303 : */
304 0 : if (file) {
305 0 : seq_pad(m, ' ');
306 : /*
307 : * If user named this anon shared memory via
308 : * prctl(PR_SET_VMA ..., use the provided name.
309 : */
310 : if (anon_name)
311 : seq_printf(m, "[anon_shmem:%s]", anon_name->name);
312 : else
313 0 : seq_file_path(m, file, "\n");
314 : goto done;
315 : }
316 :
317 0 : if (vma->vm_ops && vma->vm_ops->name) {
318 0 : name = vma->vm_ops->name(vma);
319 0 : if (name)
320 : goto done;
321 : }
322 :
323 0 : name = arch_vma_name(vma);
324 0 : if (!name) {
325 0 : if (!mm) {
326 : name = "[vdso]";
327 : goto done;
328 : }
329 :
330 0 : if (vma->vm_start <= mm->brk &&
331 0 : vma->vm_end >= mm->start_brk) {
332 : name = "[heap]";
333 : goto done;
334 : }
335 :
336 0 : if (is_stack(vma)) {
337 : name = "[stack]";
338 : goto done;
339 : }
340 :
341 : if (anon_name) {
342 : seq_pad(m, ' ');
343 : seq_printf(m, "[anon:%s]", anon_name->name);
344 : }
345 : }
346 :
347 : done:
348 0 : if (name) {
349 0 : seq_pad(m, ' ');
350 0 : seq_puts(m, name);
351 : }
352 0 : seq_putc(m, '\n');
353 0 : }
354 :
355 0 : static int show_map(struct seq_file *m, void *v)
356 : {
357 0 : show_map_vma(m, v);
358 0 : return 0;
359 : }
360 :
361 : static const struct seq_operations proc_pid_maps_op = {
362 : .start = m_start,
363 : .next = m_next,
364 : .stop = m_stop,
365 : .show = show_map
366 : };
367 :
368 0 : static int pid_maps_open(struct inode *inode, struct file *file)
369 : {
370 0 : return do_maps_open(inode, file, &proc_pid_maps_op);
371 : }
372 :
373 : const struct file_operations proc_pid_maps_operations = {
374 : .open = pid_maps_open,
375 : .read = seq_read,
376 : .llseek = seq_lseek,
377 : .release = proc_map_release,
378 : };
379 :
380 : /*
381 : * Proportional Set Size(PSS): my share of RSS.
382 : *
383 : * PSS of a process is the count of pages it has in memory, where each
384 : * page is divided by the number of processes sharing it. So if a
385 : * process has 1000 pages all to itself, and 1000 shared with one other
386 : * process, its PSS will be 1500.
387 : *
388 : * To keep (accumulated) division errors low, we adopt a 64bit
389 : * fixed-point pss counter to minimize division errors. So (pss >>
390 : * PSS_SHIFT) would be the real byte count.
391 : *
392 : * A shift of 12 before division means (assuming 4K page size):
393 : * - 1M 3-user-pages add up to 8KB errors;
394 : * - supports mapcount up to 2^24, or 16M;
395 : * - supports PSS up to 2^52 bytes, or 4PB.
396 : */
397 : #define PSS_SHIFT 12
398 :
399 : #ifdef CONFIG_PROC_PAGE_MONITOR
400 : struct mem_size_stats {
401 : unsigned long resident;
402 : unsigned long shared_clean;
403 : unsigned long shared_dirty;
404 : unsigned long private_clean;
405 : unsigned long private_dirty;
406 : unsigned long referenced;
407 : unsigned long anonymous;
408 : unsigned long lazyfree;
409 : unsigned long anonymous_thp;
410 : unsigned long shmem_thp;
411 : unsigned long file_thp;
412 : unsigned long swap;
413 : unsigned long shared_hugetlb;
414 : unsigned long private_hugetlb;
415 : u64 pss;
416 : u64 pss_anon;
417 : u64 pss_file;
418 : u64 pss_shmem;
419 : u64 pss_dirty;
420 : u64 pss_locked;
421 : u64 swap_pss;
422 : };
423 :
424 0 : static void smaps_page_accumulate(struct mem_size_stats *mss,
425 : struct page *page, unsigned long size, unsigned long pss,
426 : bool dirty, bool locked, bool private)
427 : {
428 0 : mss->pss += pss;
429 :
430 0 : if (PageAnon(page))
431 0 : mss->pss_anon += pss;
432 0 : else if (PageSwapBacked(page))
433 0 : mss->pss_shmem += pss;
434 : else
435 0 : mss->pss_file += pss;
436 :
437 0 : if (locked)
438 0 : mss->pss_locked += pss;
439 :
440 0 : if (dirty || PageDirty(page)) {
441 0 : mss->pss_dirty += pss;
442 0 : if (private)
443 0 : mss->private_dirty += size;
444 : else
445 0 : mss->shared_dirty += size;
446 : } else {
447 0 : if (private)
448 0 : mss->private_clean += size;
449 : else
450 0 : mss->shared_clean += size;
451 : }
452 0 : }
453 :
454 0 : static void smaps_account(struct mem_size_stats *mss, struct page *page,
455 : bool compound, bool young, bool dirty, bool locked,
456 : bool migration)
457 : {
458 0 : int i, nr = compound ? compound_nr(page) : 1;
459 0 : unsigned long size = nr * PAGE_SIZE;
460 :
461 : /*
462 : * First accumulate quantities that depend only on |size| and the type
463 : * of the compound page.
464 : */
465 0 : if (PageAnon(page)) {
466 0 : mss->anonymous += size;
467 0 : if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
468 0 : mss->lazyfree += size;
469 : }
470 :
471 0 : mss->resident += size;
472 : /* Accumulate the size in pages that have been accessed. */
473 0 : if (young || page_is_young(page) || PageReferenced(page))
474 0 : mss->referenced += size;
475 :
476 : /*
477 : * Then accumulate quantities that may depend on sharing, or that may
478 : * differ page-by-page.
479 : *
480 : * page_count(page) == 1 guarantees the page is mapped exactly once.
481 : * If any subpage of the compound page mapped with PTE it would elevate
482 : * page_count().
483 : *
484 : * The page_mapcount() is called to get a snapshot of the mapcount.
485 : * Without holding the page lock this snapshot can be slightly wrong as
486 : * we cannot always read the mapcount atomically. It is not safe to
487 : * call page_mapcount() even with PTL held if the page is not mapped,
488 : * especially for migration entries. Treat regular migration entries
489 : * as mapcount == 1.
490 : */
491 0 : if ((page_count(page) == 1) || migration) {
492 0 : smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty,
493 : locked, true);
494 0 : return;
495 : }
496 0 : for (i = 0; i < nr; i++, page++) {
497 0 : int mapcount = page_mapcount(page);
498 0 : unsigned long pss = PAGE_SIZE << PSS_SHIFT;
499 0 : if (mapcount >= 2)
500 0 : pss /= mapcount;
501 0 : smaps_page_accumulate(mss, page, PAGE_SIZE, pss, dirty, locked,
502 : mapcount < 2);
503 : }
504 : }
505 :
506 : #ifdef CONFIG_SHMEM
507 0 : static int smaps_pte_hole(unsigned long addr, unsigned long end,
508 : __always_unused int depth, struct mm_walk *walk)
509 : {
510 0 : struct mem_size_stats *mss = walk->private;
511 0 : struct vm_area_struct *vma = walk->vma;
512 :
513 0 : mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping,
514 : linear_page_index(vma, addr),
515 : linear_page_index(vma, end));
516 :
517 0 : return 0;
518 : }
519 : #else
520 : #define smaps_pte_hole NULL
521 : #endif /* CONFIG_SHMEM */
522 :
523 : static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk)
524 : {
525 : #ifdef CONFIG_SHMEM
526 0 : if (walk->ops->pte_hole) {
527 : /* depth is not used */
528 0 : smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk);
529 : }
530 : #endif
531 : }
532 :
533 0 : static void smaps_pte_entry(pte_t *pte, unsigned long addr,
534 : struct mm_walk *walk)
535 : {
536 0 : struct mem_size_stats *mss = walk->private;
537 0 : struct vm_area_struct *vma = walk->vma;
538 0 : bool locked = !!(vma->vm_flags & VM_LOCKED);
539 0 : struct page *page = NULL;
540 0 : bool migration = false, young = false, dirty = false;
541 :
542 0 : if (pte_present(*pte)) {
543 0 : page = vm_normal_page(vma, addr, *pte);
544 0 : young = pte_young(*pte);
545 0 : dirty = pte_dirty(*pte);
546 0 : } else if (is_swap_pte(*pte)) {
547 0 : swp_entry_t swpent = pte_to_swp_entry(*pte);
548 :
549 0 : if (!non_swap_entry(swpent)) {
550 : int mapcount;
551 :
552 0 : mss->swap += PAGE_SIZE;
553 0 : mapcount = swp_swapcount(swpent);
554 0 : if (mapcount >= 2) {
555 0 : u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
556 :
557 0 : do_div(pss_delta, mapcount);
558 0 : mss->swap_pss += pss_delta;
559 : } else {
560 0 : mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
561 : }
562 0 : } else if (is_pfn_swap_entry(swpent)) {
563 0 : if (is_migration_entry(swpent))
564 0 : migration = true;
565 0 : page = pfn_swap_entry_to_page(swpent);
566 : }
567 : } else {
568 : smaps_pte_hole_lookup(addr, walk);
569 : return;
570 : }
571 :
572 0 : if (!page)
573 : return;
574 :
575 0 : smaps_account(mss, page, false, young, dirty, locked, migration);
576 : }
577 :
578 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
579 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
580 : struct mm_walk *walk)
581 : {
582 : struct mem_size_stats *mss = walk->private;
583 : struct vm_area_struct *vma = walk->vma;
584 : bool locked = !!(vma->vm_flags & VM_LOCKED);
585 : struct page *page = NULL;
586 : bool migration = false;
587 :
588 : if (pmd_present(*pmd)) {
589 : /* FOLL_DUMP will return -EFAULT on huge zero page */
590 : page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
591 : } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) {
592 : swp_entry_t entry = pmd_to_swp_entry(*pmd);
593 :
594 : if (is_migration_entry(entry)) {
595 : migration = true;
596 : page = pfn_swap_entry_to_page(entry);
597 : }
598 : }
599 : if (IS_ERR_OR_NULL(page))
600 : return;
601 : if (PageAnon(page))
602 : mss->anonymous_thp += HPAGE_PMD_SIZE;
603 : else if (PageSwapBacked(page))
604 : mss->shmem_thp += HPAGE_PMD_SIZE;
605 : else if (is_zone_device_page(page))
606 : /* pass */;
607 : else
608 : mss->file_thp += HPAGE_PMD_SIZE;
609 :
610 : smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd),
611 : locked, migration);
612 : }
613 : #else
614 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
615 : struct mm_walk *walk)
616 : {
617 : }
618 : #endif
619 :
620 0 : static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
621 : struct mm_walk *walk)
622 : {
623 0 : struct vm_area_struct *vma = walk->vma;
624 : pte_t *pte;
625 : spinlock_t *ptl;
626 :
627 0 : ptl = pmd_trans_huge_lock(pmd, vma);
628 : if (ptl) {
629 : smaps_pmd_entry(pmd, addr, walk);
630 : spin_unlock(ptl);
631 : goto out;
632 : }
633 :
634 0 : if (pmd_trans_unstable(pmd))
635 : goto out;
636 : /*
637 : * The mmap_lock held all the way back in m_start() is what
638 : * keeps khugepaged out of here and from collapsing things
639 : * in here.
640 : */
641 0 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
642 0 : for (; addr != end; pte++, addr += PAGE_SIZE)
643 0 : smaps_pte_entry(pte, addr, walk);
644 0 : pte_unmap_unlock(pte - 1, ptl);
645 : out:
646 0 : cond_resched();
647 0 : return 0;
648 : }
649 :
650 0 : static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
651 : {
652 : /*
653 : * Don't forget to update Documentation/ on changes.
654 : */
655 : static const char mnemonics[BITS_PER_LONG][2] = {
656 : /*
657 : * In case if we meet a flag we don't know about.
658 : */
659 : [0 ... (BITS_PER_LONG-1)] = "??",
660 :
661 : [ilog2(VM_READ)] = "rd",
662 : [ilog2(VM_WRITE)] = "wr",
663 : [ilog2(VM_EXEC)] = "ex",
664 : [ilog2(VM_SHARED)] = "sh",
665 : [ilog2(VM_MAYREAD)] = "mr",
666 : [ilog2(VM_MAYWRITE)] = "mw",
667 : [ilog2(VM_MAYEXEC)] = "me",
668 : [ilog2(VM_MAYSHARE)] = "ms",
669 : [ilog2(VM_GROWSDOWN)] = "gd",
670 : [ilog2(VM_PFNMAP)] = "pf",
671 : [ilog2(VM_LOCKED)] = "lo",
672 : [ilog2(VM_IO)] = "io",
673 : [ilog2(VM_SEQ_READ)] = "sr",
674 : [ilog2(VM_RAND_READ)] = "rr",
675 : [ilog2(VM_DONTCOPY)] = "dc",
676 : [ilog2(VM_DONTEXPAND)] = "de",
677 : [ilog2(VM_LOCKONFAULT)] = "lf",
678 : [ilog2(VM_ACCOUNT)] = "ac",
679 : [ilog2(VM_NORESERVE)] = "nr",
680 : [ilog2(VM_HUGETLB)] = "ht",
681 : [ilog2(VM_SYNC)] = "sf",
682 : [ilog2(VM_ARCH_1)] = "ar",
683 : [ilog2(VM_WIPEONFORK)] = "wf",
684 : [ilog2(VM_DONTDUMP)] = "dd",
685 : #ifdef CONFIG_ARM64_BTI
686 : [ilog2(VM_ARM64_BTI)] = "bt",
687 : #endif
688 : #ifdef CONFIG_MEM_SOFT_DIRTY
689 : [ilog2(VM_SOFTDIRTY)] = "sd",
690 : #endif
691 : [ilog2(VM_MIXEDMAP)] = "mm",
692 : [ilog2(VM_HUGEPAGE)] = "hg",
693 : [ilog2(VM_NOHUGEPAGE)] = "nh",
694 : [ilog2(VM_MERGEABLE)] = "mg",
695 : [ilog2(VM_UFFD_MISSING)]= "um",
696 : [ilog2(VM_UFFD_WP)] = "uw",
697 : #ifdef CONFIG_ARM64_MTE
698 : [ilog2(VM_MTE)] = "mt",
699 : [ilog2(VM_MTE_ALLOWED)] = "",
700 : #endif
701 : #ifdef CONFIG_ARCH_HAS_PKEYS
702 : /* These come out via ProtectionKey: */
703 : [ilog2(VM_PKEY_BIT0)] = "",
704 : [ilog2(VM_PKEY_BIT1)] = "",
705 : [ilog2(VM_PKEY_BIT2)] = "",
706 : [ilog2(VM_PKEY_BIT3)] = "",
707 : #if VM_PKEY_BIT4
708 : [ilog2(VM_PKEY_BIT4)] = "",
709 : #endif
710 : #endif /* CONFIG_ARCH_HAS_PKEYS */
711 : #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
712 : [ilog2(VM_UFFD_MINOR)] = "ui",
713 : #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
714 : };
715 : size_t i;
716 :
717 0 : seq_puts(m, "VmFlags: ");
718 0 : for (i = 0; i < BITS_PER_LONG; i++) {
719 0 : if (!mnemonics[i][0])
720 0 : continue;
721 0 : if (vma->vm_flags & (1UL << i)) {
722 0 : seq_putc(m, mnemonics[i][0]);
723 0 : seq_putc(m, mnemonics[i][1]);
724 0 : seq_putc(m, ' ');
725 : }
726 : }
727 0 : seq_putc(m, '\n');
728 0 : }
729 :
730 : #ifdef CONFIG_HUGETLB_PAGE
731 : static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
732 : unsigned long addr, unsigned long end,
733 : struct mm_walk *walk)
734 : {
735 : struct mem_size_stats *mss = walk->private;
736 : struct vm_area_struct *vma = walk->vma;
737 : struct page *page = NULL;
738 :
739 : if (pte_present(*pte)) {
740 : page = vm_normal_page(vma, addr, *pte);
741 : } else if (is_swap_pte(*pte)) {
742 : swp_entry_t swpent = pte_to_swp_entry(*pte);
743 :
744 : if (is_pfn_swap_entry(swpent))
745 : page = pfn_swap_entry_to_page(swpent);
746 : }
747 : if (page) {
748 : if (page_mapcount(page) >= 2 || hugetlb_pmd_shared(pte))
749 : mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
750 : else
751 : mss->private_hugetlb += huge_page_size(hstate_vma(vma));
752 : }
753 : return 0;
754 : }
755 : #else
756 : #define smaps_hugetlb_range NULL
757 : #endif /* HUGETLB_PAGE */
758 :
759 : static const struct mm_walk_ops smaps_walk_ops = {
760 : .pmd_entry = smaps_pte_range,
761 : .hugetlb_entry = smaps_hugetlb_range,
762 : };
763 :
764 : static const struct mm_walk_ops smaps_shmem_walk_ops = {
765 : .pmd_entry = smaps_pte_range,
766 : .hugetlb_entry = smaps_hugetlb_range,
767 : .pte_hole = smaps_pte_hole,
768 : };
769 :
770 : /*
771 : * Gather mem stats from @vma with the indicated beginning
772 : * address @start, and keep them in @mss.
773 : *
774 : * Use vm_start of @vma as the beginning address if @start is 0.
775 : */
776 0 : static void smap_gather_stats(struct vm_area_struct *vma,
777 : struct mem_size_stats *mss, unsigned long start)
778 : {
779 0 : const struct mm_walk_ops *ops = &smaps_walk_ops;
780 :
781 : /* Invalid start */
782 0 : if (start >= vma->vm_end)
783 : return;
784 :
785 : #ifdef CONFIG_SHMEM
786 0 : if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
787 : /*
788 : * For shared or readonly shmem mappings we know that all
789 : * swapped out pages belong to the shmem object, and we can
790 : * obtain the swap value much more efficiently. For private
791 : * writable mappings, we might have COW pages that are
792 : * not affected by the parent swapped out pages of the shmem
793 : * object, so we have to distinguish them during the page walk.
794 : * Unless we know that the shmem object (or the part mapped by
795 : * our VMA) has no swapped out pages at all.
796 : */
797 0 : unsigned long shmem_swapped = shmem_swap_usage(vma);
798 :
799 0 : if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
800 : !(vma->vm_flags & VM_WRITE))) {
801 0 : mss->swap += shmem_swapped;
802 : } else {
803 : ops = &smaps_shmem_walk_ops;
804 : }
805 : }
806 : #endif
807 : /* mmap_lock is held in m_start */
808 0 : if (!start)
809 0 : walk_page_vma(vma, ops, mss);
810 : else
811 0 : walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
812 : }
813 :
814 : #define SEQ_PUT_DEC(str, val) \
815 : seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
816 :
817 : /* Show the contents common for smaps and smaps_rollup */
818 0 : static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
819 : bool rollup_mode)
820 : {
821 0 : SEQ_PUT_DEC("Rss: ", mss->resident);
822 0 : SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
823 0 : SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT);
824 0 : if (rollup_mode) {
825 : /*
826 : * These are meaningful only for smaps_rollup, otherwise two of
827 : * them are zero, and the other one is the same as Pss.
828 : */
829 0 : SEQ_PUT_DEC(" kB\nPss_Anon: ",
830 : mss->pss_anon >> PSS_SHIFT);
831 0 : SEQ_PUT_DEC(" kB\nPss_File: ",
832 : mss->pss_file >> PSS_SHIFT);
833 0 : SEQ_PUT_DEC(" kB\nPss_Shmem: ",
834 : mss->pss_shmem >> PSS_SHIFT);
835 : }
836 0 : SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
837 0 : SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
838 0 : SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
839 0 : SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
840 0 : SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
841 0 : SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
842 0 : SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
843 0 : SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
844 0 : SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
845 0 : SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp);
846 0 : SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
847 0 : seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
848 0 : mss->private_hugetlb >> 10, 7);
849 0 : SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
850 0 : SEQ_PUT_DEC(" kB\nSwapPss: ",
851 : mss->swap_pss >> PSS_SHIFT);
852 0 : SEQ_PUT_DEC(" kB\nLocked: ",
853 : mss->pss_locked >> PSS_SHIFT);
854 0 : seq_puts(m, " kB\n");
855 0 : }
856 :
857 0 : static int show_smap(struct seq_file *m, void *v)
858 : {
859 0 : struct vm_area_struct *vma = v;
860 : struct mem_size_stats mss;
861 :
862 0 : memset(&mss, 0, sizeof(mss));
863 :
864 0 : smap_gather_stats(vma, &mss, 0);
865 :
866 0 : show_map_vma(m, vma);
867 :
868 0 : SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
869 0 : SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
870 0 : SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
871 0 : seq_puts(m, " kB\n");
872 :
873 0 : __show_smap(m, &mss, false);
874 :
875 0 : seq_printf(m, "THPeligible: %d\n",
876 0 : hugepage_vma_check(vma, vma->vm_flags, true, false, true));
877 :
878 : if (arch_pkeys_enabled())
879 : seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
880 0 : show_smap_vma_flags(m, vma);
881 :
882 0 : return 0;
883 : }
884 :
885 0 : static int show_smaps_rollup(struct seq_file *m, void *v)
886 : {
887 0 : struct proc_maps_private *priv = m->private;
888 : struct mem_size_stats mss;
889 0 : struct mm_struct *mm = priv->mm;
890 : struct vm_area_struct *vma;
891 0 : unsigned long vma_start = 0, last_vma_end = 0;
892 0 : int ret = 0;
893 0 : VMA_ITERATOR(vmi, mm, 0);
894 :
895 0 : priv->task = get_proc_task(priv->inode);
896 0 : if (!priv->task)
897 : return -ESRCH;
898 :
899 0 : if (!mm || !mmget_not_zero(mm)) {
900 : ret = -ESRCH;
901 : goto out_put_task;
902 : }
903 :
904 0 : memset(&mss, 0, sizeof(mss));
905 :
906 0 : ret = mmap_read_lock_killable(mm);
907 0 : if (ret)
908 : goto out_put_mm;
909 :
910 0 : hold_task_mempolicy(priv);
911 0 : vma = vma_next(&vmi);
912 :
913 0 : if (unlikely(!vma))
914 : goto empty_set;
915 :
916 0 : vma_start = vma->vm_start;
917 : do {
918 0 : smap_gather_stats(vma, &mss, 0);
919 0 : last_vma_end = vma->vm_end;
920 :
921 : /*
922 : * Release mmap_lock temporarily if someone wants to
923 : * access it for write request.
924 : */
925 0 : if (mmap_lock_is_contended(mm)) {
926 0 : vma_iter_invalidate(&vmi);
927 0 : mmap_read_unlock(mm);
928 0 : ret = mmap_read_lock_killable(mm);
929 0 : if (ret) {
930 : release_task_mempolicy(priv);
931 : goto out_put_mm;
932 : }
933 :
934 : /*
935 : * After dropping the lock, there are four cases to
936 : * consider. See the following example for explanation.
937 : *
938 : * +------+------+-----------+
939 : * | VMA1 | VMA2 | VMA3 |
940 : * +------+------+-----------+
941 : * | | | |
942 : * 4k 8k 16k 400k
943 : *
944 : * Suppose we drop the lock after reading VMA2 due to
945 : * contention, then we get:
946 : *
947 : * last_vma_end = 16k
948 : *
949 : * 1) VMA2 is freed, but VMA3 exists:
950 : *
951 : * vma_next(vmi) will return VMA3.
952 : * In this case, just continue from VMA3.
953 : *
954 : * 2) VMA2 still exists:
955 : *
956 : * vma_next(vmi) will return VMA3.
957 : * In this case, just continue from VMA3.
958 : *
959 : * 3) No more VMAs can be found:
960 : *
961 : * vma_next(vmi) will return NULL.
962 : * No more things to do, just break.
963 : *
964 : * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
965 : *
966 : * vma_next(vmi) will return VMA' whose range
967 : * contains last_vma_end.
968 : * Iterate VMA' from last_vma_end.
969 : */
970 0 : vma = vma_next(&vmi);
971 : /* Case 3 above */
972 0 : if (!vma)
973 : break;
974 :
975 : /* Case 1 and 2 above */
976 0 : if (vma->vm_start >= last_vma_end)
977 0 : continue;
978 :
979 : /* Case 4 above */
980 0 : if (vma->vm_end > last_vma_end)
981 0 : smap_gather_stats(vma, &mss, last_vma_end);
982 : }
983 0 : } for_each_vma(vmi, vma);
984 :
985 : empty_set:
986 0 : show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0);
987 0 : seq_pad(m, ' ');
988 0 : seq_puts(m, "[rollup]\n");
989 :
990 0 : __show_smap(m, &mss, true);
991 :
992 0 : release_task_mempolicy(priv);
993 : mmap_read_unlock(mm);
994 :
995 : out_put_mm:
996 0 : mmput(mm);
997 : out_put_task:
998 0 : put_task_struct(priv->task);
999 0 : priv->task = NULL;
1000 :
1001 0 : return ret;
1002 : }
1003 : #undef SEQ_PUT_DEC
1004 :
1005 : static const struct seq_operations proc_pid_smaps_op = {
1006 : .start = m_start,
1007 : .next = m_next,
1008 : .stop = m_stop,
1009 : .show = show_smap
1010 : };
1011 :
1012 0 : static int pid_smaps_open(struct inode *inode, struct file *file)
1013 : {
1014 0 : return do_maps_open(inode, file, &proc_pid_smaps_op);
1015 : }
1016 :
1017 0 : static int smaps_rollup_open(struct inode *inode, struct file *file)
1018 : {
1019 : int ret;
1020 : struct proc_maps_private *priv;
1021 :
1022 0 : priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
1023 0 : if (!priv)
1024 : return -ENOMEM;
1025 :
1026 0 : ret = single_open(file, show_smaps_rollup, priv);
1027 0 : if (ret)
1028 : goto out_free;
1029 :
1030 0 : priv->inode = inode;
1031 0 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
1032 0 : if (IS_ERR(priv->mm)) {
1033 0 : ret = PTR_ERR(priv->mm);
1034 :
1035 0 : single_release(inode, file);
1036 0 : goto out_free;
1037 : }
1038 :
1039 : return 0;
1040 :
1041 : out_free:
1042 0 : kfree(priv);
1043 0 : return ret;
1044 : }
1045 :
1046 0 : static int smaps_rollup_release(struct inode *inode, struct file *file)
1047 : {
1048 0 : struct seq_file *seq = file->private_data;
1049 0 : struct proc_maps_private *priv = seq->private;
1050 :
1051 0 : if (priv->mm)
1052 0 : mmdrop(priv->mm);
1053 :
1054 0 : kfree(priv);
1055 0 : return single_release(inode, file);
1056 : }
1057 :
1058 : const struct file_operations proc_pid_smaps_operations = {
1059 : .open = pid_smaps_open,
1060 : .read = seq_read,
1061 : .llseek = seq_lseek,
1062 : .release = proc_map_release,
1063 : };
1064 :
1065 : const struct file_operations proc_pid_smaps_rollup_operations = {
1066 : .open = smaps_rollup_open,
1067 : .read = seq_read,
1068 : .llseek = seq_lseek,
1069 : .release = smaps_rollup_release,
1070 : };
1071 :
1072 : enum clear_refs_types {
1073 : CLEAR_REFS_ALL = 1,
1074 : CLEAR_REFS_ANON,
1075 : CLEAR_REFS_MAPPED,
1076 : CLEAR_REFS_SOFT_DIRTY,
1077 : CLEAR_REFS_MM_HIWATER_RSS,
1078 : CLEAR_REFS_LAST,
1079 : };
1080 :
1081 : struct clear_refs_private {
1082 : enum clear_refs_types type;
1083 : };
1084 :
1085 : #ifdef CONFIG_MEM_SOFT_DIRTY
1086 :
1087 : static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1088 : {
1089 : struct page *page;
1090 :
1091 : if (!pte_write(pte))
1092 : return false;
1093 : if (!is_cow_mapping(vma->vm_flags))
1094 : return false;
1095 : if (likely(!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)))
1096 : return false;
1097 : page = vm_normal_page(vma, addr, pte);
1098 : if (!page)
1099 : return false;
1100 : return page_maybe_dma_pinned(page);
1101 : }
1102 :
1103 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1104 : unsigned long addr, pte_t *pte)
1105 : {
1106 : /*
1107 : * The soft-dirty tracker uses #PF-s to catch writes
1108 : * to pages, so write-protect the pte as well. See the
1109 : * Documentation/admin-guide/mm/soft-dirty.rst for full description
1110 : * of how soft-dirty works.
1111 : */
1112 : pte_t ptent = *pte;
1113 :
1114 : if (pte_present(ptent)) {
1115 : pte_t old_pte;
1116 :
1117 : if (pte_is_pinned(vma, addr, ptent))
1118 : return;
1119 : old_pte = ptep_modify_prot_start(vma, addr, pte);
1120 : ptent = pte_wrprotect(old_pte);
1121 : ptent = pte_clear_soft_dirty(ptent);
1122 : ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
1123 : } else if (is_swap_pte(ptent)) {
1124 : ptent = pte_swp_clear_soft_dirty(ptent);
1125 : set_pte_at(vma->vm_mm, addr, pte, ptent);
1126 : }
1127 : }
1128 : #else
1129 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1130 : unsigned long addr, pte_t *pte)
1131 : {
1132 : }
1133 : #endif
1134 :
1135 : #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1136 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1137 : unsigned long addr, pmd_t *pmdp)
1138 : {
1139 : pmd_t old, pmd = *pmdp;
1140 :
1141 : if (pmd_present(pmd)) {
1142 : /* See comment in change_huge_pmd() */
1143 : old = pmdp_invalidate(vma, addr, pmdp);
1144 : if (pmd_dirty(old))
1145 : pmd = pmd_mkdirty(pmd);
1146 : if (pmd_young(old))
1147 : pmd = pmd_mkyoung(pmd);
1148 :
1149 : pmd = pmd_wrprotect(pmd);
1150 : pmd = pmd_clear_soft_dirty(pmd);
1151 :
1152 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1153 : } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
1154 : pmd = pmd_swp_clear_soft_dirty(pmd);
1155 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1156 : }
1157 : }
1158 : #else
1159 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1160 : unsigned long addr, pmd_t *pmdp)
1161 : {
1162 : }
1163 : #endif
1164 :
1165 0 : static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
1166 : unsigned long end, struct mm_walk *walk)
1167 : {
1168 0 : struct clear_refs_private *cp = walk->private;
1169 0 : struct vm_area_struct *vma = walk->vma;
1170 : pte_t *pte, ptent;
1171 : spinlock_t *ptl;
1172 : struct page *page;
1173 :
1174 0 : ptl = pmd_trans_huge_lock(pmd, vma);
1175 : if (ptl) {
1176 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1177 : clear_soft_dirty_pmd(vma, addr, pmd);
1178 : goto out;
1179 : }
1180 :
1181 : if (!pmd_present(*pmd))
1182 : goto out;
1183 :
1184 : page = pmd_page(*pmd);
1185 :
1186 : /* Clear accessed and referenced bits. */
1187 : pmdp_test_and_clear_young(vma, addr, pmd);
1188 : test_and_clear_page_young(page);
1189 : ClearPageReferenced(page);
1190 : out:
1191 : spin_unlock(ptl);
1192 : return 0;
1193 : }
1194 :
1195 0 : if (pmd_trans_unstable(pmd))
1196 : return 0;
1197 :
1198 0 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1199 0 : for (; addr != end; pte++, addr += PAGE_SIZE) {
1200 0 : ptent = *pte;
1201 :
1202 0 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1203 0 : clear_soft_dirty(vma, addr, pte);
1204 0 : continue;
1205 : }
1206 :
1207 0 : if (!pte_present(ptent))
1208 0 : continue;
1209 :
1210 0 : page = vm_normal_page(vma, addr, ptent);
1211 0 : if (!page)
1212 0 : continue;
1213 :
1214 : /* Clear accessed and referenced bits. */
1215 0 : ptep_test_and_clear_young(vma, addr, pte);
1216 0 : test_and_clear_page_young(page);
1217 : ClearPageReferenced(page);
1218 : }
1219 0 : pte_unmap_unlock(pte - 1, ptl);
1220 0 : cond_resched();
1221 : return 0;
1222 : }
1223 :
1224 0 : static int clear_refs_test_walk(unsigned long start, unsigned long end,
1225 : struct mm_walk *walk)
1226 : {
1227 0 : struct clear_refs_private *cp = walk->private;
1228 0 : struct vm_area_struct *vma = walk->vma;
1229 :
1230 0 : if (vma->vm_flags & VM_PFNMAP)
1231 : return 1;
1232 :
1233 : /*
1234 : * Writing 1 to /proc/pid/clear_refs affects all pages.
1235 : * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1236 : * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1237 : * Writing 4 to /proc/pid/clear_refs affects all pages.
1238 : */
1239 0 : if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1240 : return 1;
1241 0 : if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1242 : return 1;
1243 0 : return 0;
1244 : }
1245 :
1246 : static const struct mm_walk_ops clear_refs_walk_ops = {
1247 : .pmd_entry = clear_refs_pte_range,
1248 : .test_walk = clear_refs_test_walk,
1249 : };
1250 :
1251 0 : static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1252 : size_t count, loff_t *ppos)
1253 : {
1254 : struct task_struct *task;
1255 : char buffer[PROC_NUMBUF];
1256 : struct mm_struct *mm;
1257 : struct vm_area_struct *vma;
1258 : enum clear_refs_types type;
1259 : int itype;
1260 : int rv;
1261 :
1262 0 : memset(buffer, 0, sizeof(buffer));
1263 0 : if (count > sizeof(buffer) - 1)
1264 0 : count = sizeof(buffer) - 1;
1265 0 : if (copy_from_user(buffer, buf, count))
1266 : return -EFAULT;
1267 0 : rv = kstrtoint(strstrip(buffer), 10, &itype);
1268 0 : if (rv < 0)
1269 0 : return rv;
1270 0 : type = (enum clear_refs_types)itype;
1271 0 : if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1272 : return -EINVAL;
1273 :
1274 0 : task = get_proc_task(file_inode(file));
1275 0 : if (!task)
1276 : return -ESRCH;
1277 0 : mm = get_task_mm(task);
1278 0 : if (mm) {
1279 0 : VMA_ITERATOR(vmi, mm, 0);
1280 : struct mmu_notifier_range range;
1281 0 : struct clear_refs_private cp = {
1282 : .type = type,
1283 : };
1284 :
1285 0 : if (mmap_write_lock_killable(mm)) {
1286 : count = -EINTR;
1287 : goto out_mm;
1288 : }
1289 0 : if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1290 : /*
1291 : * Writing 5 to /proc/pid/clear_refs resets the peak
1292 : * resident set size to this mm's current rss value.
1293 : */
1294 : reset_mm_hiwater_rss(mm);
1295 : goto out_unlock;
1296 : }
1297 :
1298 0 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1299 0 : for_each_vma(vmi, vma) {
1300 : if (!(vma->vm_flags & VM_SOFTDIRTY))
1301 0 : continue;
1302 : vm_flags_clear(vma, VM_SOFTDIRTY);
1303 : vma_set_page_prot(vma);
1304 : }
1305 :
1306 0 : inc_tlb_flush_pending(mm);
1307 : mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
1308 : 0, mm, 0, -1UL);
1309 : mmu_notifier_invalidate_range_start(&range);
1310 : }
1311 0 : walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp);
1312 0 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1313 0 : mmu_notifier_invalidate_range_end(&range);
1314 0 : flush_tlb_mm(mm);
1315 : dec_tlb_flush_pending(mm);
1316 : }
1317 : out_unlock:
1318 : mmap_write_unlock(mm);
1319 : out_mm:
1320 0 : mmput(mm);
1321 : }
1322 0 : put_task_struct(task);
1323 :
1324 0 : return count;
1325 : }
1326 :
1327 : const struct file_operations proc_clear_refs_operations = {
1328 : .write = clear_refs_write,
1329 : .llseek = noop_llseek,
1330 : };
1331 :
1332 : typedef struct {
1333 : u64 pme;
1334 : } pagemap_entry_t;
1335 :
1336 : struct pagemapread {
1337 : int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
1338 : pagemap_entry_t *buffer;
1339 : bool show_pfn;
1340 : };
1341 :
1342 : #define PAGEMAP_WALK_SIZE (PMD_SIZE)
1343 : #define PAGEMAP_WALK_MASK (PMD_MASK)
1344 :
1345 : #define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1346 : #define PM_PFRAME_BITS 55
1347 : #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1348 : #define PM_SOFT_DIRTY BIT_ULL(55)
1349 : #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1350 : #define PM_UFFD_WP BIT_ULL(57)
1351 : #define PM_FILE BIT_ULL(61)
1352 : #define PM_SWAP BIT_ULL(62)
1353 : #define PM_PRESENT BIT_ULL(63)
1354 :
1355 : #define PM_END_OF_BUFFER 1
1356 :
1357 : static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1358 : {
1359 0 : return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1360 : }
1361 :
1362 : static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1363 : struct pagemapread *pm)
1364 : {
1365 0 : pm->buffer[pm->pos++] = *pme;
1366 0 : if (pm->pos >= pm->len)
1367 : return PM_END_OF_BUFFER;
1368 : return 0;
1369 : }
1370 :
1371 0 : static int pagemap_pte_hole(unsigned long start, unsigned long end,
1372 : __always_unused int depth, struct mm_walk *walk)
1373 : {
1374 0 : struct pagemapread *pm = walk->private;
1375 0 : unsigned long addr = start;
1376 0 : int err = 0;
1377 :
1378 0 : while (addr < end) {
1379 0 : struct vm_area_struct *vma = find_vma(walk->mm, addr);
1380 0 : pagemap_entry_t pme = make_pme(0, 0);
1381 : /* End of address space hole, which we mark as non-present. */
1382 : unsigned long hole_end;
1383 :
1384 0 : if (vma)
1385 0 : hole_end = min(end, vma->vm_start);
1386 : else
1387 : hole_end = end;
1388 :
1389 0 : for (; addr < hole_end; addr += PAGE_SIZE) {
1390 0 : err = add_to_pagemap(addr, &pme, pm);
1391 0 : if (err)
1392 : goto out;
1393 : }
1394 :
1395 0 : if (!vma)
1396 : break;
1397 :
1398 : /* Addresses in the VMA. */
1399 : if (vma->vm_flags & VM_SOFTDIRTY)
1400 : pme = make_pme(0, PM_SOFT_DIRTY);
1401 0 : for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1402 0 : err = add_to_pagemap(addr, &pme, pm);
1403 0 : if (err)
1404 : goto out;
1405 : }
1406 : }
1407 : out:
1408 0 : return err;
1409 : }
1410 :
1411 0 : static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1412 : struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1413 : {
1414 0 : u64 frame = 0, flags = 0;
1415 0 : struct page *page = NULL;
1416 0 : bool migration = false;
1417 :
1418 0 : if (pte_present(pte)) {
1419 0 : if (pm->show_pfn)
1420 0 : frame = pte_pfn(pte);
1421 0 : flags |= PM_PRESENT;
1422 0 : page = vm_normal_page(vma, addr, pte);
1423 0 : if (pte_soft_dirty(pte))
1424 : flags |= PM_SOFT_DIRTY;
1425 : if (pte_uffd_wp(pte))
1426 : flags |= PM_UFFD_WP;
1427 0 : } else if (is_swap_pte(pte)) {
1428 : swp_entry_t entry;
1429 0 : if (pte_swp_soft_dirty(pte))
1430 : flags |= PM_SOFT_DIRTY;
1431 : if (pte_swp_uffd_wp(pte))
1432 : flags |= PM_UFFD_WP;
1433 0 : entry = pte_to_swp_entry(pte);
1434 0 : if (pm->show_pfn) {
1435 : pgoff_t offset;
1436 : /*
1437 : * For PFN swap offsets, keeping the offset field
1438 : * to be PFN only to be compatible with old smaps.
1439 : */
1440 0 : if (is_pfn_swap_entry(entry))
1441 0 : offset = swp_offset_pfn(entry);
1442 : else
1443 0 : offset = swp_offset(entry);
1444 0 : frame = swp_type(entry) |
1445 0 : (offset << MAX_SWAPFILES_SHIFT);
1446 : }
1447 0 : flags |= PM_SWAP;
1448 0 : migration = is_migration_entry(entry);
1449 0 : if (is_pfn_swap_entry(entry))
1450 0 : page = pfn_swap_entry_to_page(entry);
1451 : if (pte_marker_entry_uffd_wp(entry))
1452 : flags |= PM_UFFD_WP;
1453 : }
1454 :
1455 0 : if (page && !PageAnon(page))
1456 0 : flags |= PM_FILE;
1457 0 : if (page && !migration && page_mapcount(page) == 1)
1458 0 : flags |= PM_MMAP_EXCLUSIVE;
1459 : if (vma->vm_flags & VM_SOFTDIRTY)
1460 : flags |= PM_SOFT_DIRTY;
1461 :
1462 0 : return make_pme(frame, flags);
1463 : }
1464 :
1465 0 : static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1466 : struct mm_walk *walk)
1467 : {
1468 0 : struct vm_area_struct *vma = walk->vma;
1469 0 : struct pagemapread *pm = walk->private;
1470 : spinlock_t *ptl;
1471 : pte_t *pte, *orig_pte;
1472 0 : int err = 0;
1473 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1474 : bool migration = false;
1475 :
1476 : ptl = pmd_trans_huge_lock(pmdp, vma);
1477 : if (ptl) {
1478 : u64 flags = 0, frame = 0;
1479 : pmd_t pmd = *pmdp;
1480 : struct page *page = NULL;
1481 :
1482 : if (vma->vm_flags & VM_SOFTDIRTY)
1483 : flags |= PM_SOFT_DIRTY;
1484 :
1485 : if (pmd_present(pmd)) {
1486 : page = pmd_page(pmd);
1487 :
1488 : flags |= PM_PRESENT;
1489 : if (pmd_soft_dirty(pmd))
1490 : flags |= PM_SOFT_DIRTY;
1491 : if (pmd_uffd_wp(pmd))
1492 : flags |= PM_UFFD_WP;
1493 : if (pm->show_pfn)
1494 : frame = pmd_pfn(pmd) +
1495 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1496 : }
1497 : #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1498 : else if (is_swap_pmd(pmd)) {
1499 : swp_entry_t entry = pmd_to_swp_entry(pmd);
1500 : unsigned long offset;
1501 :
1502 : if (pm->show_pfn) {
1503 : if (is_pfn_swap_entry(entry))
1504 : offset = swp_offset_pfn(entry);
1505 : else
1506 : offset = swp_offset(entry);
1507 : offset = offset +
1508 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1509 : frame = swp_type(entry) |
1510 : (offset << MAX_SWAPFILES_SHIFT);
1511 : }
1512 : flags |= PM_SWAP;
1513 : if (pmd_swp_soft_dirty(pmd))
1514 : flags |= PM_SOFT_DIRTY;
1515 : if (pmd_swp_uffd_wp(pmd))
1516 : flags |= PM_UFFD_WP;
1517 : VM_BUG_ON(!is_pmd_migration_entry(pmd));
1518 : migration = is_migration_entry(entry);
1519 : page = pfn_swap_entry_to_page(entry);
1520 : }
1521 : #endif
1522 :
1523 : if (page && !migration && page_mapcount(page) == 1)
1524 : flags |= PM_MMAP_EXCLUSIVE;
1525 :
1526 : for (; addr != end; addr += PAGE_SIZE) {
1527 : pagemap_entry_t pme = make_pme(frame, flags);
1528 :
1529 : err = add_to_pagemap(addr, &pme, pm);
1530 : if (err)
1531 : break;
1532 : if (pm->show_pfn) {
1533 : if (flags & PM_PRESENT)
1534 : frame++;
1535 : else if (flags & PM_SWAP)
1536 : frame += (1 << MAX_SWAPFILES_SHIFT);
1537 : }
1538 : }
1539 : spin_unlock(ptl);
1540 : return err;
1541 : }
1542 :
1543 : if (pmd_trans_unstable(pmdp))
1544 : return 0;
1545 : #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1546 :
1547 : /*
1548 : * We can assume that @vma always points to a valid one and @end never
1549 : * goes beyond vma->vm_end.
1550 : */
1551 0 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1552 0 : for (; addr < end; pte++, addr += PAGE_SIZE) {
1553 : pagemap_entry_t pme;
1554 :
1555 0 : pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1556 0 : err = add_to_pagemap(addr, &pme, pm);
1557 0 : if (err)
1558 : break;
1559 : }
1560 0 : pte_unmap_unlock(orig_pte, ptl);
1561 :
1562 0 : cond_resched();
1563 :
1564 0 : return err;
1565 : }
1566 :
1567 : #ifdef CONFIG_HUGETLB_PAGE
1568 : /* This function walks within one hugetlb entry in the single call */
1569 : static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1570 : unsigned long addr, unsigned long end,
1571 : struct mm_walk *walk)
1572 : {
1573 : struct pagemapread *pm = walk->private;
1574 : struct vm_area_struct *vma = walk->vma;
1575 : u64 flags = 0, frame = 0;
1576 : int err = 0;
1577 : pte_t pte;
1578 :
1579 : if (vma->vm_flags & VM_SOFTDIRTY)
1580 : flags |= PM_SOFT_DIRTY;
1581 :
1582 : pte = huge_ptep_get(ptep);
1583 : if (pte_present(pte)) {
1584 : struct page *page = pte_page(pte);
1585 :
1586 : if (!PageAnon(page))
1587 : flags |= PM_FILE;
1588 :
1589 : if (page_mapcount(page) == 1)
1590 : flags |= PM_MMAP_EXCLUSIVE;
1591 :
1592 : if (huge_pte_uffd_wp(pte))
1593 : flags |= PM_UFFD_WP;
1594 :
1595 : flags |= PM_PRESENT;
1596 : if (pm->show_pfn)
1597 : frame = pte_pfn(pte) +
1598 : ((addr & ~hmask) >> PAGE_SHIFT);
1599 : } else if (pte_swp_uffd_wp_any(pte)) {
1600 : flags |= PM_UFFD_WP;
1601 : }
1602 :
1603 : for (; addr != end; addr += PAGE_SIZE) {
1604 : pagemap_entry_t pme = make_pme(frame, flags);
1605 :
1606 : err = add_to_pagemap(addr, &pme, pm);
1607 : if (err)
1608 : return err;
1609 : if (pm->show_pfn && (flags & PM_PRESENT))
1610 : frame++;
1611 : }
1612 :
1613 : cond_resched();
1614 :
1615 : return err;
1616 : }
1617 : #else
1618 : #define pagemap_hugetlb_range NULL
1619 : #endif /* HUGETLB_PAGE */
1620 :
1621 : static const struct mm_walk_ops pagemap_ops = {
1622 : .pmd_entry = pagemap_pmd_range,
1623 : .pte_hole = pagemap_pte_hole,
1624 : .hugetlb_entry = pagemap_hugetlb_range,
1625 : };
1626 :
1627 : /*
1628 : * /proc/pid/pagemap - an array mapping virtual pages to pfns
1629 : *
1630 : * For each page in the address space, this file contains one 64-bit entry
1631 : * consisting of the following:
1632 : *
1633 : * Bits 0-54 page frame number (PFN) if present
1634 : * Bits 0-4 swap type if swapped
1635 : * Bits 5-54 swap offset if swapped
1636 : * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1637 : * Bit 56 page exclusively mapped
1638 : * Bit 57 pte is uffd-wp write-protected
1639 : * Bits 58-60 zero
1640 : * Bit 61 page is file-page or shared-anon
1641 : * Bit 62 page swapped
1642 : * Bit 63 page present
1643 : *
1644 : * If the page is not present but in swap, then the PFN contains an
1645 : * encoding of the swap file number and the page's offset into the
1646 : * swap. Unmapped pages return a null PFN. This allows determining
1647 : * precisely which pages are mapped (or in swap) and comparing mapped
1648 : * pages between processes.
1649 : *
1650 : * Efficient users of this interface will use /proc/pid/maps to
1651 : * determine which areas of memory are actually mapped and llseek to
1652 : * skip over unmapped regions.
1653 : */
1654 0 : static ssize_t pagemap_read(struct file *file, char __user *buf,
1655 : size_t count, loff_t *ppos)
1656 : {
1657 0 : struct mm_struct *mm = file->private_data;
1658 : struct pagemapread pm;
1659 : unsigned long src;
1660 : unsigned long svpfn;
1661 : unsigned long start_vaddr;
1662 : unsigned long end_vaddr;
1663 0 : int ret = 0, copied = 0;
1664 :
1665 0 : if (!mm || !mmget_not_zero(mm))
1666 : goto out;
1667 :
1668 0 : ret = -EINVAL;
1669 : /* file position must be aligned */
1670 0 : if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1671 : goto out_mm;
1672 :
1673 0 : ret = 0;
1674 0 : if (!count)
1675 : goto out_mm;
1676 :
1677 : /* do not disclose physical addresses: attack vector */
1678 0 : pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1679 :
1680 0 : pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1681 0 : pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1682 0 : ret = -ENOMEM;
1683 0 : if (!pm.buffer)
1684 : goto out_mm;
1685 :
1686 0 : src = *ppos;
1687 0 : svpfn = src / PM_ENTRY_BYTES;
1688 0 : end_vaddr = mm->task_size;
1689 :
1690 : /* watch out for wraparound */
1691 0 : start_vaddr = end_vaddr;
1692 0 : if (svpfn <= (ULONG_MAX >> PAGE_SHIFT))
1693 0 : start_vaddr = untagged_addr(svpfn << PAGE_SHIFT);
1694 :
1695 : /* Ensure the address is inside the task */
1696 0 : if (start_vaddr > mm->task_size)
1697 0 : start_vaddr = end_vaddr;
1698 :
1699 : /*
1700 : * The odds are that this will stop walking way
1701 : * before end_vaddr, because the length of the
1702 : * user buffer is tracked in "pm", and the walk
1703 : * will stop when we hit the end of the buffer.
1704 : */
1705 : ret = 0;
1706 0 : while (count && (start_vaddr < end_vaddr)) {
1707 : int len;
1708 : unsigned long end;
1709 :
1710 0 : pm.pos = 0;
1711 0 : end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1712 : /* overflow ? */
1713 0 : if (end < start_vaddr || end > end_vaddr)
1714 0 : end = end_vaddr;
1715 0 : ret = mmap_read_lock_killable(mm);
1716 0 : if (ret)
1717 : goto out_free;
1718 0 : ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm);
1719 0 : mmap_read_unlock(mm);
1720 0 : start_vaddr = end;
1721 :
1722 0 : len = min(count, PM_ENTRY_BYTES * pm.pos);
1723 0 : if (copy_to_user(buf, pm.buffer, len)) {
1724 : ret = -EFAULT;
1725 : goto out_free;
1726 : }
1727 0 : copied += len;
1728 0 : buf += len;
1729 0 : count -= len;
1730 : }
1731 0 : *ppos += copied;
1732 0 : if (!ret || ret == PM_END_OF_BUFFER)
1733 0 : ret = copied;
1734 :
1735 : out_free:
1736 0 : kfree(pm.buffer);
1737 : out_mm:
1738 0 : mmput(mm);
1739 : out:
1740 0 : return ret;
1741 : }
1742 :
1743 0 : static int pagemap_open(struct inode *inode, struct file *file)
1744 : {
1745 : struct mm_struct *mm;
1746 :
1747 0 : mm = proc_mem_open(inode, PTRACE_MODE_READ);
1748 0 : if (IS_ERR(mm))
1749 0 : return PTR_ERR(mm);
1750 0 : file->private_data = mm;
1751 0 : return 0;
1752 : }
1753 :
1754 0 : static int pagemap_release(struct inode *inode, struct file *file)
1755 : {
1756 0 : struct mm_struct *mm = file->private_data;
1757 :
1758 0 : if (mm)
1759 : mmdrop(mm);
1760 0 : return 0;
1761 : }
1762 :
1763 : const struct file_operations proc_pagemap_operations = {
1764 : .llseek = mem_lseek, /* borrow this */
1765 : .read = pagemap_read,
1766 : .open = pagemap_open,
1767 : .release = pagemap_release,
1768 : };
1769 : #endif /* CONFIG_PROC_PAGE_MONITOR */
1770 :
1771 : #ifdef CONFIG_NUMA
1772 :
1773 : struct numa_maps {
1774 : unsigned long pages;
1775 : unsigned long anon;
1776 : unsigned long active;
1777 : unsigned long writeback;
1778 : unsigned long mapcount_max;
1779 : unsigned long dirty;
1780 : unsigned long swapcache;
1781 : unsigned long node[MAX_NUMNODES];
1782 : };
1783 :
1784 : struct numa_maps_private {
1785 : struct proc_maps_private proc_maps;
1786 : struct numa_maps md;
1787 : };
1788 :
1789 : static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1790 : unsigned long nr_pages)
1791 : {
1792 : int count = page_mapcount(page);
1793 :
1794 : md->pages += nr_pages;
1795 : if (pte_dirty || PageDirty(page))
1796 : md->dirty += nr_pages;
1797 :
1798 : if (PageSwapCache(page))
1799 : md->swapcache += nr_pages;
1800 :
1801 : if (PageActive(page) || PageUnevictable(page))
1802 : md->active += nr_pages;
1803 :
1804 : if (PageWriteback(page))
1805 : md->writeback += nr_pages;
1806 :
1807 : if (PageAnon(page))
1808 : md->anon += nr_pages;
1809 :
1810 : if (count > md->mapcount_max)
1811 : md->mapcount_max = count;
1812 :
1813 : md->node[page_to_nid(page)] += nr_pages;
1814 : }
1815 :
1816 : static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1817 : unsigned long addr)
1818 : {
1819 : struct page *page;
1820 : int nid;
1821 :
1822 : if (!pte_present(pte))
1823 : return NULL;
1824 :
1825 : page = vm_normal_page(vma, addr, pte);
1826 : if (!page || is_zone_device_page(page))
1827 : return NULL;
1828 :
1829 : if (PageReserved(page))
1830 : return NULL;
1831 :
1832 : nid = page_to_nid(page);
1833 : if (!node_isset(nid, node_states[N_MEMORY]))
1834 : return NULL;
1835 :
1836 : return page;
1837 : }
1838 :
1839 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1840 : static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1841 : struct vm_area_struct *vma,
1842 : unsigned long addr)
1843 : {
1844 : struct page *page;
1845 : int nid;
1846 :
1847 : if (!pmd_present(pmd))
1848 : return NULL;
1849 :
1850 : page = vm_normal_page_pmd(vma, addr, pmd);
1851 : if (!page)
1852 : return NULL;
1853 :
1854 : if (PageReserved(page))
1855 : return NULL;
1856 :
1857 : nid = page_to_nid(page);
1858 : if (!node_isset(nid, node_states[N_MEMORY]))
1859 : return NULL;
1860 :
1861 : return page;
1862 : }
1863 : #endif
1864 :
1865 : static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1866 : unsigned long end, struct mm_walk *walk)
1867 : {
1868 : struct numa_maps *md = walk->private;
1869 : struct vm_area_struct *vma = walk->vma;
1870 : spinlock_t *ptl;
1871 : pte_t *orig_pte;
1872 : pte_t *pte;
1873 :
1874 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1875 : ptl = pmd_trans_huge_lock(pmd, vma);
1876 : if (ptl) {
1877 : struct page *page;
1878 :
1879 : page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1880 : if (page)
1881 : gather_stats(page, md, pmd_dirty(*pmd),
1882 : HPAGE_PMD_SIZE/PAGE_SIZE);
1883 : spin_unlock(ptl);
1884 : return 0;
1885 : }
1886 :
1887 : if (pmd_trans_unstable(pmd))
1888 : return 0;
1889 : #endif
1890 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1891 : do {
1892 : struct page *page = can_gather_numa_stats(*pte, vma, addr);
1893 : if (!page)
1894 : continue;
1895 : gather_stats(page, md, pte_dirty(*pte), 1);
1896 :
1897 : } while (pte++, addr += PAGE_SIZE, addr != end);
1898 : pte_unmap_unlock(orig_pte, ptl);
1899 : cond_resched();
1900 : return 0;
1901 : }
1902 : #ifdef CONFIG_HUGETLB_PAGE
1903 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1904 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1905 : {
1906 : pte_t huge_pte = huge_ptep_get(pte);
1907 : struct numa_maps *md;
1908 : struct page *page;
1909 :
1910 : if (!pte_present(huge_pte))
1911 : return 0;
1912 :
1913 : page = pte_page(huge_pte);
1914 :
1915 : md = walk->private;
1916 : gather_stats(page, md, pte_dirty(huge_pte), 1);
1917 : return 0;
1918 : }
1919 :
1920 : #else
1921 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1922 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1923 : {
1924 : return 0;
1925 : }
1926 : #endif
1927 :
1928 : static const struct mm_walk_ops show_numa_ops = {
1929 : .hugetlb_entry = gather_hugetlb_stats,
1930 : .pmd_entry = gather_pte_stats,
1931 : };
1932 :
1933 : /*
1934 : * Display pages allocated per node and memory policy via /proc.
1935 : */
1936 : static int show_numa_map(struct seq_file *m, void *v)
1937 : {
1938 : struct numa_maps_private *numa_priv = m->private;
1939 : struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1940 : struct vm_area_struct *vma = v;
1941 : struct numa_maps *md = &numa_priv->md;
1942 : struct file *file = vma->vm_file;
1943 : struct mm_struct *mm = vma->vm_mm;
1944 : struct mempolicy *pol;
1945 : char buffer[64];
1946 : int nid;
1947 :
1948 : if (!mm)
1949 : return 0;
1950 :
1951 : /* Ensure we start with an empty set of numa_maps statistics. */
1952 : memset(md, 0, sizeof(*md));
1953 :
1954 : pol = __get_vma_policy(vma, vma->vm_start);
1955 : if (pol) {
1956 : mpol_to_str(buffer, sizeof(buffer), pol);
1957 : mpol_cond_put(pol);
1958 : } else {
1959 : mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1960 : }
1961 :
1962 : seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1963 :
1964 : if (file) {
1965 : seq_puts(m, " file=");
1966 : seq_file_path(m, file, "\n\t= ");
1967 : } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1968 : seq_puts(m, " heap");
1969 : } else if (is_stack(vma)) {
1970 : seq_puts(m, " stack");
1971 : }
1972 :
1973 : if (is_vm_hugetlb_page(vma))
1974 : seq_puts(m, " huge");
1975 :
1976 : /* mmap_lock is held by m_start */
1977 : walk_page_vma(vma, &show_numa_ops, md);
1978 :
1979 : if (!md->pages)
1980 : goto out;
1981 :
1982 : if (md->anon)
1983 : seq_printf(m, " anon=%lu", md->anon);
1984 :
1985 : if (md->dirty)
1986 : seq_printf(m, " dirty=%lu", md->dirty);
1987 :
1988 : if (md->pages != md->anon && md->pages != md->dirty)
1989 : seq_printf(m, " mapped=%lu", md->pages);
1990 :
1991 : if (md->mapcount_max > 1)
1992 : seq_printf(m, " mapmax=%lu", md->mapcount_max);
1993 :
1994 : if (md->swapcache)
1995 : seq_printf(m, " swapcache=%lu", md->swapcache);
1996 :
1997 : if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1998 : seq_printf(m, " active=%lu", md->active);
1999 :
2000 : if (md->writeback)
2001 : seq_printf(m, " writeback=%lu", md->writeback);
2002 :
2003 : for_each_node_state(nid, N_MEMORY)
2004 : if (md->node[nid])
2005 : seq_printf(m, " N%d=%lu", nid, md->node[nid]);
2006 :
2007 : seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
2008 : out:
2009 : seq_putc(m, '\n');
2010 : return 0;
2011 : }
2012 :
2013 : static const struct seq_operations proc_pid_numa_maps_op = {
2014 : .start = m_start,
2015 : .next = m_next,
2016 : .stop = m_stop,
2017 : .show = show_numa_map,
2018 : };
2019 :
2020 : static int pid_numa_maps_open(struct inode *inode, struct file *file)
2021 : {
2022 : return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
2023 : sizeof(struct numa_maps_private));
2024 : }
2025 :
2026 : const struct file_operations proc_pid_numa_maps_operations = {
2027 : .open = pid_numa_maps_open,
2028 : .read = seq_read,
2029 : .llseek = seq_lseek,
2030 : .release = proc_map_release,
2031 : };
2032 :
2033 : #endif /* CONFIG_NUMA */
|
