Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : #include <linux/pagewalk.h>
3 : #include <linux/mm_inline.h>
4 : #include <linux/hugetlb.h>
5 : #include <linux/huge_mm.h>
6 : #include <linux/mount.h>
7 : #include <linux/seq_file.h>
8 : #include <linux/highmem.h>
9 : #include <linux/ptrace.h>
10 : #include <linux/slab.h>
11 : #include <linux/pagemap.h>
12 : #include <linux/mempolicy.h>
13 : #include <linux/rmap.h>
14 : #include <linux/swap.h>
15 : #include <linux/sched/mm.h>
16 : #include <linux/swapops.h>
17 : #include <linux/mmu_notifier.h>
18 : #include <linux/page_idle.h>
19 : #include <linux/shmem_fs.h>
20 : #include <linux/uaccess.h>
21 : #include <linux/pkeys.h>
22 :
23 : #include <asm/elf.h>
24 : #include <asm/tlb.h>
25 : #include <asm/tlbflush.h>
26 : #include "internal.h"
27 :
28 : #define SEQ_PUT_DEC(str, val) \
29 : seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
30 0 : void task_mem(struct seq_file *m, struct mm_struct *mm)
31 : {
32 : unsigned long text, lib, swap, anon, file, shmem;
33 : unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
34 :
35 0 : anon = get_mm_counter(mm, MM_ANONPAGES);
36 0 : file = get_mm_counter(mm, MM_FILEPAGES);
37 0 : shmem = get_mm_counter(mm, MM_SHMEMPAGES);
38 :
39 : /*
40 : * Note: to minimize their overhead, mm maintains hiwater_vm and
41 : * hiwater_rss only when about to *lower* total_vm or rss. Any
42 : * collector of these hiwater stats must therefore get total_vm
43 : * and rss too, which will usually be the higher. Barriers? not
44 : * worth the effort, such snapshots can always be inconsistent.
45 : */
46 0 : hiwater_vm = total_vm = mm->total_vm;
47 0 : if (hiwater_vm < mm->hiwater_vm)
48 0 : hiwater_vm = mm->hiwater_vm;
49 0 : hiwater_rss = total_rss = anon + file + shmem;
50 0 : if (hiwater_rss < mm->hiwater_rss)
51 0 : hiwater_rss = mm->hiwater_rss;
52 :
53 : /* split executable areas between text and lib */
54 0 : text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
55 0 : text = min(text, mm->exec_vm << PAGE_SHIFT);
56 0 : lib = (mm->exec_vm << PAGE_SHIFT) - text;
57 :
58 0 : swap = get_mm_counter(mm, MM_SWAPENTS);
59 0 : SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
60 0 : SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
61 0 : SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
62 0 : SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
63 0 : SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
64 0 : SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
65 0 : SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
66 0 : SEQ_PUT_DEC(" kB\nRssFile:\t", file);
67 0 : SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
68 0 : SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
69 0 : SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
70 0 : seq_put_decimal_ull_width(m,
71 0 : " kB\nVmExe:\t", text >> 10, 8);
72 0 : seq_put_decimal_ull_width(m,
73 0 : " kB\nVmLib:\t", lib >> 10, 8);
74 0 : seq_put_decimal_ull_width(m,
75 0 : " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
76 0 : SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
77 0 : seq_puts(m, " kB\n");
78 0 : hugetlb_report_usage(m, mm);
79 0 : }
80 : #undef SEQ_PUT_DEC
81 :
82 0 : unsigned long task_vsize(struct mm_struct *mm)
83 : {
84 0 : return PAGE_SIZE * mm->total_vm;
85 : }
86 :
87 0 : unsigned long task_statm(struct mm_struct *mm,
88 : unsigned long *shared, unsigned long *text,
89 : unsigned long *data, unsigned long *resident)
90 : {
91 0 : *shared = get_mm_counter(mm, MM_FILEPAGES) +
92 0 : get_mm_counter(mm, MM_SHMEMPAGES);
93 0 : *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
94 0 : >> PAGE_SHIFT;
95 0 : *data = mm->data_vm + mm->stack_vm;
96 0 : *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
97 0 : return mm->total_vm;
98 : }
99 :
100 : #ifdef CONFIG_NUMA
101 : /*
102 : * Save get_task_policy() for show_numa_map().
103 : */
104 : static void hold_task_mempolicy(struct proc_maps_private *priv)
105 : {
106 : struct task_struct *task = priv->task;
107 :
108 : task_lock(task);
109 : priv->task_mempolicy = get_task_policy(task);
110 : mpol_get(priv->task_mempolicy);
111 : task_unlock(task);
112 : }
113 : static void release_task_mempolicy(struct proc_maps_private *priv)
114 : {
115 : mpol_put(priv->task_mempolicy);
116 : }
117 : #else
118 : static void hold_task_mempolicy(struct proc_maps_private *priv)
119 : {
120 : }
121 : static void release_task_mempolicy(struct proc_maps_private *priv)
122 : {
123 : }
124 : #endif
125 :
126 : static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv,
127 : loff_t *ppos)
128 : {
129 0 : struct vm_area_struct *vma = vma_next(&priv->iter);
130 :
131 0 : if (vma) {
132 0 : *ppos = vma->vm_start;
133 : } else {
134 0 : *ppos = -2UL;
135 0 : vma = get_gate_vma(priv->mm);
136 : }
137 :
138 : return vma;
139 : }
140 :
141 0 : static void *m_start(struct seq_file *m, loff_t *ppos)
142 : {
143 0 : struct proc_maps_private *priv = m->private;
144 0 : unsigned long last_addr = *ppos;
145 : struct mm_struct *mm;
146 :
147 : /* See m_next(). Zero at the start or after lseek. */
148 0 : if (last_addr == -1UL)
149 : return NULL;
150 :
151 0 : priv->task = get_proc_task(priv->inode);
152 0 : if (!priv->task)
153 : return ERR_PTR(-ESRCH);
154 :
155 0 : mm = priv->mm;
156 0 : if (!mm || !mmget_not_zero(mm)) {
157 0 : put_task_struct(priv->task);
158 0 : priv->task = NULL;
159 0 : return NULL;
160 : }
161 :
162 0 : if (mmap_read_lock_killable(mm)) {
163 0 : mmput(mm);
164 0 : put_task_struct(priv->task);
165 0 : priv->task = NULL;
166 0 : return ERR_PTR(-EINTR);
167 : }
168 :
169 0 : vma_iter_init(&priv->iter, mm, last_addr);
170 0 : hold_task_mempolicy(priv);
171 0 : if (last_addr == -2UL)
172 : return get_gate_vma(mm);
173 :
174 : return proc_get_vma(priv, ppos);
175 : }
176 :
177 0 : static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
178 : {
179 0 : if (*ppos == -2UL) {
180 0 : *ppos = -1UL;
181 0 : return NULL;
182 : }
183 0 : return proc_get_vma(m->private, ppos);
184 : }
185 :
186 0 : static void m_stop(struct seq_file *m, void *v)
187 : {
188 0 : struct proc_maps_private *priv = m->private;
189 0 : struct mm_struct *mm = priv->mm;
190 :
191 0 : if (!priv->task)
192 : return;
193 :
194 0 : release_task_mempolicy(priv);
195 0 : mmap_read_unlock(mm);
196 0 : mmput(mm);
197 0 : put_task_struct(priv->task);
198 0 : priv->task = NULL;
199 : }
200 :
201 0 : static int proc_maps_open(struct inode *inode, struct file *file,
202 : const struct seq_operations *ops, int psize)
203 : {
204 0 : struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
205 :
206 0 : if (!priv)
207 : return -ENOMEM;
208 :
209 0 : priv->inode = inode;
210 0 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
211 0 : if (IS_ERR(priv->mm)) {
212 0 : int err = PTR_ERR(priv->mm);
213 :
214 0 : seq_release_private(inode, file);
215 0 : return err;
216 : }
217 :
218 : return 0;
219 : }
220 :
221 0 : static int proc_map_release(struct inode *inode, struct file *file)
222 : {
223 0 : struct seq_file *seq = file->private_data;
224 0 : struct proc_maps_private *priv = seq->private;
225 :
226 0 : if (priv->mm)
227 0 : mmdrop(priv->mm);
228 :
229 0 : return seq_release_private(inode, file);
230 : }
231 :
232 : static int do_maps_open(struct inode *inode, struct file *file,
233 : const struct seq_operations *ops)
234 : {
235 0 : return proc_maps_open(inode, file, ops,
236 : sizeof(struct proc_maps_private));
237 : }
238 :
239 : /*
240 : * Indicate if the VMA is a stack for the given task; for
241 : * /proc/PID/maps that is the stack of the main task.
242 : */
243 : static int is_stack(struct vm_area_struct *vma)
244 : {
245 : /*
246 : * We make no effort to guess what a given thread considers to be
247 : * its "stack". It's not even well-defined for programs written
248 : * languages like Go.
249 : */
250 0 : return vma->vm_start <= vma->vm_mm->start_stack &&
251 0 : vma->vm_end >= vma->vm_mm->start_stack;
252 : }
253 :
254 0 : static void show_vma_header_prefix(struct seq_file *m,
255 : unsigned long start, unsigned long end,
256 : vm_flags_t flags, unsigned long long pgoff,
257 : dev_t dev, unsigned long ino)
258 : {
259 0 : seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
260 0 : seq_put_hex_ll(m, NULL, start, 8);
261 0 : seq_put_hex_ll(m, "-", end, 8);
262 0 : seq_putc(m, ' ');
263 0 : seq_putc(m, flags & VM_READ ? 'r' : '-');
264 0 : seq_putc(m, flags & VM_WRITE ? 'w' : '-');
265 0 : seq_putc(m, flags & VM_EXEC ? 'x' : '-');
266 0 : seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
267 0 : seq_put_hex_ll(m, " ", pgoff, 8);
268 0 : seq_put_hex_ll(m, " ", MAJOR(dev), 2);
269 0 : seq_put_hex_ll(m, ":", MINOR(dev), 2);
270 0 : seq_put_decimal_ull(m, " ", ino);
271 0 : seq_putc(m, ' ');
272 0 : }
273 :
274 : static void
275 0 : show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
276 : {
277 0 : struct anon_vma_name *anon_name = NULL;
278 0 : struct mm_struct *mm = vma->vm_mm;
279 0 : struct file *file = vma->vm_file;
280 0 : vm_flags_t flags = vma->vm_flags;
281 0 : unsigned long ino = 0;
282 0 : unsigned long long pgoff = 0;
283 : unsigned long start, end;
284 0 : dev_t dev = 0;
285 0 : const char *name = NULL;
286 :
287 0 : if (file) {
288 0 : struct inode *inode = file_inode(vma->vm_file);
289 0 : dev = inode->i_sb->s_dev;
290 0 : ino = inode->i_ino;
291 0 : pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
292 : }
293 :
294 0 : start = vma->vm_start;
295 0 : end = vma->vm_end;
296 0 : show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
297 : if (mm)
298 : anon_name = anon_vma_name(vma);
299 :
300 : /*
301 : * Print the dentry name for named mappings, and a
302 : * special [heap] marker for the heap:
303 : */
304 0 : if (file) {
305 0 : seq_pad(m, ' ');
306 : /*
307 : * If user named this anon shared memory via
308 : * prctl(PR_SET_VMA ..., use the provided name.
309 : */
310 : if (anon_name)
311 : seq_printf(m, "[anon_shmem:%s]", anon_name->name);
312 : else
313 0 : seq_file_path(m, file, "\n");
314 : goto done;
315 : }
316 :
317 0 : if (vma->vm_ops && vma->vm_ops->name) {
318 0 : name = vma->vm_ops->name(vma);
319 0 : if (name)
320 : goto done;
321 : }
322 :
323 0 : name = arch_vma_name(vma);
324 0 : if (!name) {
325 0 : if (!mm) {
326 : name = "[vdso]";
327 : goto done;
328 : }
329 :
330 0 : if (vma->vm_start <= mm->brk &&
331 0 : vma->vm_end >= mm->start_brk) {
332 : name = "[heap]";
333 : goto done;
334 : }
335 :
336 0 : if (is_stack(vma)) {
337 : name = "[stack]";
338 : goto done;
339 : }
340 :
341 : if (anon_name) {
342 : seq_pad(m, ' ');
343 : seq_printf(m, "[anon:%s]", anon_name->name);
344 : }
345 : }
346 :
347 : done:
348 0 : if (name) {
349 0 : seq_pad(m, ' ');
350 0 : seq_puts(m, name);
351 : }
352 0 : seq_putc(m, '\n');
353 0 : }
354 :
355 0 : static int show_map(struct seq_file *m, void *v)
356 : {
357 0 : show_map_vma(m, v);
358 0 : return 0;
359 : }
360 :
361 : static const struct seq_operations proc_pid_maps_op = {
362 : .start = m_start,
363 : .next = m_next,
364 : .stop = m_stop,
365 : .show = show_map
366 : };
367 :
368 0 : static int pid_maps_open(struct inode *inode, struct file *file)
369 : {
370 0 : return do_maps_open(inode, file, &proc_pid_maps_op);
371 : }
372 :
373 : const struct file_operations proc_pid_maps_operations = {
374 : .open = pid_maps_open,
375 : .read = seq_read,
376 : .llseek = seq_lseek,
377 : .release = proc_map_release,
378 : };
379 :
380 : /*
381 : * Proportional Set Size(PSS): my share of RSS.
382 : *
383 : * PSS of a process is the count of pages it has in memory, where each
384 : * page is divided by the number of processes sharing it. So if a
385 : * process has 1000 pages all to itself, and 1000 shared with one other
386 : * process, its PSS will be 1500.
387 : *
388 : * To keep (accumulated) division errors low, we adopt a 64bit
389 : * fixed-point pss counter to minimize division errors. So (pss >>
390 : * PSS_SHIFT) would be the real byte count.
391 : *
392 : * A shift of 12 before division means (assuming 4K page size):
393 : * - 1M 3-user-pages add up to 8KB errors;
394 : * - supports mapcount up to 2^24, or 16M;
395 : * - supports PSS up to 2^52 bytes, or 4PB.
396 : */
397 : #define PSS_SHIFT 12
398 :
399 : #ifdef CONFIG_PROC_PAGE_MONITOR
400 : struct mem_size_stats {
401 : unsigned long resident;
402 : unsigned long shared_clean;
403 : unsigned long shared_dirty;
404 : unsigned long private_clean;
405 : unsigned long private_dirty;
406 : unsigned long referenced;
407 : unsigned long anonymous;
408 : unsigned long lazyfree;
409 : unsigned long anonymous_thp;
410 : unsigned long shmem_thp;
411 : unsigned long file_thp;
412 : unsigned long swap;
413 : unsigned long shared_hugetlb;
414 : unsigned long private_hugetlb;
415 : u64 pss;
416 : u64 pss_anon;
417 : u64 pss_file;
418 : u64 pss_shmem;
419 : u64 pss_dirty;
420 : u64 pss_locked;
421 : u64 swap_pss;
422 : };
423 :
424 0 : static void smaps_page_accumulate(struct mem_size_stats *mss,
425 : struct page *page, unsigned long size, unsigned long pss,
426 : bool dirty, bool locked, bool private)
427 : {
428 0 : mss->pss += pss;
429 :
430 0 : if (PageAnon(page))
431 0 : mss->pss_anon += pss;
432 0 : else if (PageSwapBacked(page))
433 0 : mss->pss_shmem += pss;
434 : else
435 0 : mss->pss_file += pss;
436 :
437 0 : if (locked)
438 0 : mss->pss_locked += pss;
439 :
440 0 : if (dirty || PageDirty(page)) {
441 0 : mss->pss_dirty += pss;
442 0 : if (private)
443 0 : mss->private_dirty += size;
444 : else
445 0 : mss->shared_dirty += size;
446 : } else {
447 0 : if (private)
448 0 : mss->private_clean += size;
449 : else
450 0 : mss->shared_clean += size;
451 : }
452 0 : }
453 :
454 0 : static void smaps_account(struct mem_size_stats *mss, struct page *page,
455 : bool compound, bool young, bool dirty, bool locked,
456 : bool migration)
457 : {
458 0 : int i, nr = compound ? compound_nr(page) : 1;
459 0 : unsigned long size = nr * PAGE_SIZE;
460 :
461 : /*
462 : * First accumulate quantities that depend only on |size| and the type
463 : * of the compound page.
464 : */
465 0 : if (PageAnon(page)) {
466 0 : mss->anonymous += size;
467 0 : if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
468 0 : mss->lazyfree += size;
469 : }
470 :
471 0 : mss->resident += size;
472 : /* Accumulate the size in pages that have been accessed. */
473 0 : if (young || page_is_young(page) || PageReferenced(page))
474 0 : mss->referenced += size;
475 :
476 : /*
477 : * Then accumulate quantities that may depend on sharing, or that may
478 : * differ page-by-page.
479 : *
480 : * page_count(page) == 1 guarantees the page is mapped exactly once.
481 : * If any subpage of the compound page mapped with PTE it would elevate
482 : * page_count().
483 : *
484 : * The page_mapcount() is called to get a snapshot of the mapcount.
485 : * Without holding the page lock this snapshot can be slightly wrong as
486 : * we cannot always read the mapcount atomically. It is not safe to
487 : * call page_mapcount() even with PTL held if the page is not mapped,
488 : * especially for migration entries. Treat regular migration entries
489 : * as mapcount == 1.
490 : */
491 0 : if ((page_count(page) == 1) || migration) {
492 0 : smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty,
493 : locked, true);
494 0 : return;
495 : }
496 0 : for (i = 0; i < nr; i++, page++) {
497 0 : int mapcount = page_mapcount(page);
498 0 : unsigned long pss = PAGE_SIZE << PSS_SHIFT;
499 0 : if (mapcount >= 2)
500 0 : pss /= mapcount;
501 0 : smaps_page_accumulate(mss, page, PAGE_SIZE, pss, dirty, locked,
502 : mapcount < 2);
503 : }
504 : }
505 :
506 : #ifdef CONFIG_SHMEM
507 0 : static int smaps_pte_hole(unsigned long addr, unsigned long end,
508 : __always_unused int depth, struct mm_walk *walk)
509 : {
510 0 : struct mem_size_stats *mss = walk->private;
511 0 : struct vm_area_struct *vma = walk->vma;
512 :
513 0 : mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping,
514 : linear_page_index(vma, addr),
515 : linear_page_index(vma, end));
516 :
517 0 : return 0;
518 : }
519 : #else
520 : #define smaps_pte_hole NULL
521 : #endif /* CONFIG_SHMEM */
522 :
523 : static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk)
524 : {
525 : #ifdef CONFIG_SHMEM
526 0 : if (walk->ops->pte_hole) {
527 : /* depth is not used */
528 0 : smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk);
529 : }
530 : #endif
531 : }
532 :
533 0 : static void smaps_pte_entry(pte_t *pte, unsigned long addr,
534 : struct mm_walk *walk)
535 : {
536 0 : struct mem_size_stats *mss = walk->private;
537 0 : struct vm_area_struct *vma = walk->vma;
538 0 : bool locked = !!(vma->vm_flags & VM_LOCKED);
539 0 : struct page *page = NULL;
540 0 : bool migration = false, young = false, dirty = false;
541 :
542 0 : if (pte_present(*pte)) {
543 0 : page = vm_normal_page(vma, addr, *pte);
544 0 : young = pte_young(*pte);
545 0 : dirty = pte_dirty(*pte);
546 0 : } else if (is_swap_pte(*pte)) {
547 0 : swp_entry_t swpent = pte_to_swp_entry(*pte);
548 :
549 0 : if (!non_swap_entry(swpent)) {
550 : int mapcount;
551 :
552 0 : mss->swap += PAGE_SIZE;
553 0 : mapcount = swp_swapcount(swpent);
554 0 : if (mapcount >= 2) {
555 0 : u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
556 :
557 0 : do_div(pss_delta, mapcount);
558 0 : mss->swap_pss += pss_delta;
559 : } else {
560 0 : mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
561 : }
562 0 : } else if (is_pfn_swap_entry(swpent)) {
563 0 : if (is_migration_entry(swpent))
564 0 : migration = true;
565 0 : page = pfn_swap_entry_to_page(swpent);
566 : }
567 : } else {
568 : smaps_pte_hole_lookup(addr, walk);
569 : return;
570 : }
571 :
572 0 : if (!page)
573 : return;
574 :
575 0 : smaps_account(mss, page, false, young, dirty, locked, migration);
576 : }
577 :
578 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
579 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
580 : struct mm_walk *walk)
581 : {
582 : struct mem_size_stats *mss = walk->private;
583 : struct vm_area_struct *vma = walk->vma;
584 : bool locked = !!(vma->vm_flags & VM_LOCKED);
585 : struct page *page = NULL;
586 : bool migration = false;
587 :
588 : if (pmd_present(*pmd)) {
589 : /* FOLL_DUMP will return -EFAULT on huge zero page */
590 : page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
591 : } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) {
592 : swp_entry_t entry = pmd_to_swp_entry(*pmd);
593 :
594 : if (is_migration_entry(entry)) {
595 : migration = true;
596 : page = pfn_swap_entry_to_page(entry);
597 : }
598 : }
599 : if (IS_ERR_OR_NULL(page))
600 : return;
601 : if (PageAnon(page))
602 : mss->anonymous_thp += HPAGE_PMD_SIZE;
603 : else if (PageSwapBacked(page))
604 : mss->shmem_thp += HPAGE_PMD_SIZE;
605 : else if (is_zone_device_page(page))
606 : /* pass */;
607 : else
608 : mss->file_thp += HPAGE_PMD_SIZE;
609 :
610 : smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd),
611 : locked, migration);
612 : }
613 : #else
614 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
615 : struct mm_walk *walk)
616 : {
617 : }
618 : #endif
619 :
620 0 : static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
621 : struct mm_walk *walk)
622 : {
623 0 : struct vm_area_struct *vma = walk->vma;
624 : pte_t *pte;
625 : spinlock_t *ptl;
626 :
627 0 : ptl = pmd_trans_huge_lock(pmd, vma);
628 : if (ptl) {
629 : smaps_pmd_entry(pmd, addr, walk);
630 : spin_unlock(ptl);
631 : goto out;
632 : }
633 :
634 0 : if (pmd_trans_unstable(pmd))
635 : goto out;
636 : /*
637 : * The mmap_lock held all the way back in m_start() is what
638 : * keeps khugepaged out of here and from collapsing things
639 : * in here.
640 : */
641 0 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
642 0 : for (; addr != end; pte++, addr += PAGE_SIZE)
643 0 : smaps_pte_entry(pte, addr, walk);
644 0 : pte_unmap_unlock(pte - 1, ptl);
645 : out:
646 0 : cond_resched();
647 0 : return 0;
648 : }
649 :
650 0 : static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
651 : {
652 : /*
653 : * Don't forget to update Documentation/ on changes.
654 : */
655 : static const char mnemonics[BITS_PER_LONG][2] = {
656 : /*
657 : * In case if we meet a flag we don't know about.
658 : */
659 : [0 ... (BITS_PER_LONG-1)] = "??",
660 :
661 : [ilog2(VM_READ)] = "rd",
662 : [ilog2(VM_WRITE)] = "wr",
663 : [ilog2(VM_EXEC)] = "ex",
664 : [ilog2(VM_SHARED)] = "sh",
665 : [ilog2(VM_MAYREAD)] = "mr",
666 : [ilog2(VM_MAYWRITE)] = "mw",
667 : [ilog2(VM_MAYEXEC)] = "me",
668 : [ilog2(VM_MAYSHARE)] = "ms",
669 : [ilog2(VM_GROWSDOWN)] = "gd",
670 : [ilog2(VM_PFNMAP)] = "pf",
671 : [ilog2(VM_LOCKED)] = "lo",
672 : [ilog2(VM_IO)] = "io",
673 : [ilog2(VM_SEQ_READ)] = "sr",
674 : [ilog2(VM_RAND_READ)] = "rr",
675 : [ilog2(VM_DONTCOPY)] = "dc",
676 : [ilog2(VM_DONTEXPAND)] = "de",
677 : [ilog2(VM_LOCKONFAULT)] = "lf",
678 : [ilog2(VM_ACCOUNT)] = "ac",
679 : [ilog2(VM_NORESERVE)] = "nr",
680 : [ilog2(VM_HUGETLB)] = "ht",
681 : [ilog2(VM_SYNC)] = "sf",
682 : [ilog2(VM_ARCH_1)] = "ar",
683 : [ilog2(VM_WIPEONFORK)] = "wf",
684 : [ilog2(VM_DONTDUMP)] = "dd",
685 : #ifdef CONFIG_ARM64_BTI
686 : [ilog2(VM_ARM64_BTI)] = "bt",
687 : #endif
688 : #ifdef CONFIG_MEM_SOFT_DIRTY
689 : [ilog2(VM_SOFTDIRTY)] = "sd",
690 : #endif
691 : [ilog2(VM_MIXEDMAP)] = "mm",
692 : [ilog2(VM_HUGEPAGE)] = "hg",
693 : [ilog2(VM_NOHUGEPAGE)] = "nh",
694 : [ilog2(VM_MERGEABLE)] = "mg",
695 : [ilog2(VM_UFFD_MISSING)]= "um",
696 : [ilog2(VM_UFFD_WP)] = "uw",
697 : #ifdef CONFIG_ARM64_MTE
698 : [ilog2(VM_MTE)] = "mt",
699 : [ilog2(VM_MTE_ALLOWED)] = "",
700 : #endif
701 : #ifdef CONFIG_ARCH_HAS_PKEYS
702 : /* These come out via ProtectionKey: */
703 : [ilog2(VM_PKEY_BIT0)] = "",
704 : [ilog2(VM_PKEY_BIT1)] = "",
705 : [ilog2(VM_PKEY_BIT2)] = "",
706 : [ilog2(VM_PKEY_BIT3)] = "",
707 : #if VM_PKEY_BIT4
708 : [ilog2(VM_PKEY_BIT4)] = "",
709 : #endif
710 : #endif /* CONFIG_ARCH_HAS_PKEYS */
711 : #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
712 : [ilog2(VM_UFFD_MINOR)] = "ui",
713 : #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
714 : };
715 : size_t i;
716 :
717 0 : seq_puts(m, "VmFlags: ");
718 0 : for (i = 0; i < BITS_PER_LONG; i++) {
719 0 : if (!mnemonics[i][0])
720 0 : continue;
721 0 : if (vma->vm_flags & (1UL << i)) {
722 0 : seq_putc(m, mnemonics[i][0]);
723 0 : seq_putc(m, mnemonics[i][1]);
724 0 : seq_putc(m, ' ');
725 : }
726 : }
727 0 : seq_putc(m, '\n');
728 0 : }
729 :
730 : #ifdef CONFIG_HUGETLB_PAGE
731 : static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
732 : unsigned long addr, unsigned long end,
733 : struct mm_walk *walk)
734 : {
735 : struct mem_size_stats *mss = walk->private;
736 : struct vm_area_struct *vma = walk->vma;
737 : struct page *page = NULL;
738 :
739 : if (pte_present(*pte)) {
740 : page = vm_normal_page(vma, addr, *pte);
741 : } else if (is_swap_pte(*pte)) {
742 : swp_entry_t swpent = pte_to_swp_entry(*pte);
743 :
744 : if (is_pfn_swap_entry(swpent))
745 : page = pfn_swap_entry_to_page(swpent);
746 : }
747 : if (page) {
748 : if (page_mapcount(page) >= 2 || hugetlb_pmd_shared(pte))
749 : mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
750 : else
751 : mss->private_hugetlb += huge_page_size(hstate_vma(vma));
752 : }
753 : return 0;
754 : }
755 : #else
756 : #define smaps_hugetlb_range NULL
757 : #endif /* HUGETLB_PAGE */
758 :
759 : static const struct mm_walk_ops smaps_walk_ops = {
760 : .pmd_entry = smaps_pte_range,
761 : .hugetlb_entry = smaps_hugetlb_range,
762 : };
763 :
764 : static const struct mm_walk_ops smaps_shmem_walk_ops = {
765 : .pmd_entry = smaps_pte_range,
766 : .hugetlb_entry = smaps_hugetlb_range,
767 : .pte_hole = smaps_pte_hole,
768 : };
769 :
770 : /*
771 : * Gather mem stats from @vma with the indicated beginning
772 : * address @start, and keep them in @mss.
773 : *
774 : * Use vm_start of @vma as the beginning address if @start is 0.
775 : */
776 0 : static void smap_gather_stats(struct vm_area_struct *vma,
777 : struct mem_size_stats *mss, unsigned long start)
778 : {
779 0 : const struct mm_walk_ops *ops = &smaps_walk_ops;
780 :
781 : /* Invalid start */
782 0 : if (start >= vma->vm_end)
783 : return;
784 :
785 0 : if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
786 : /*
787 : * For shared or readonly shmem mappings we know that all
788 : * swapped out pages belong to the shmem object, and we can
789 : * obtain the swap value much more efficiently. For private
790 : * writable mappings, we might have COW pages that are
791 : * not affected by the parent swapped out pages of the shmem
792 : * object, so we have to distinguish them during the page walk.
793 : * Unless we know that the shmem object (or the part mapped by
794 : * our VMA) has no swapped out pages at all.
795 : */
796 0 : unsigned long shmem_swapped = shmem_swap_usage(vma);
797 :
798 0 : if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
799 : !(vma->vm_flags & VM_WRITE))) {
800 0 : mss->swap += shmem_swapped;
801 : } else {
802 : ops = &smaps_shmem_walk_ops;
803 : }
804 : }
805 :
806 : /* mmap_lock is held in m_start */
807 0 : if (!start)
808 0 : walk_page_vma(vma, ops, mss);
809 : else
810 0 : walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
811 : }
812 :
813 : #define SEQ_PUT_DEC(str, val) \
814 : seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
815 :
816 : /* Show the contents common for smaps and smaps_rollup */
817 0 : static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
818 : bool rollup_mode)
819 : {
820 0 : SEQ_PUT_DEC("Rss: ", mss->resident);
821 0 : SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
822 0 : SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT);
823 0 : if (rollup_mode) {
824 : /*
825 : * These are meaningful only for smaps_rollup, otherwise two of
826 : * them are zero, and the other one is the same as Pss.
827 : */
828 0 : SEQ_PUT_DEC(" kB\nPss_Anon: ",
829 : mss->pss_anon >> PSS_SHIFT);
830 0 : SEQ_PUT_DEC(" kB\nPss_File: ",
831 : mss->pss_file >> PSS_SHIFT);
832 0 : SEQ_PUT_DEC(" kB\nPss_Shmem: ",
833 : mss->pss_shmem >> PSS_SHIFT);
834 : }
835 0 : SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
836 0 : SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
837 0 : SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
838 0 : SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
839 0 : SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
840 0 : SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
841 0 : SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
842 0 : SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
843 0 : SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
844 0 : SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp);
845 0 : SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
846 0 : seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
847 0 : mss->private_hugetlb >> 10, 7);
848 0 : SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
849 0 : SEQ_PUT_DEC(" kB\nSwapPss: ",
850 : mss->swap_pss >> PSS_SHIFT);
851 0 : SEQ_PUT_DEC(" kB\nLocked: ",
852 : mss->pss_locked >> PSS_SHIFT);
853 0 : seq_puts(m, " kB\n");
854 0 : }
855 :
856 0 : static int show_smap(struct seq_file *m, void *v)
857 : {
858 0 : struct vm_area_struct *vma = v;
859 : struct mem_size_stats mss;
860 :
861 0 : memset(&mss, 0, sizeof(mss));
862 :
863 0 : smap_gather_stats(vma, &mss, 0);
864 :
865 0 : show_map_vma(m, vma);
866 :
867 0 : SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
868 0 : SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
869 0 : SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
870 0 : seq_puts(m, " kB\n");
871 :
872 0 : __show_smap(m, &mss, false);
873 :
874 0 : seq_printf(m, "THPeligible: %d\n",
875 0 : hugepage_vma_check(vma, vma->vm_flags, true, false, true));
876 :
877 : if (arch_pkeys_enabled())
878 : seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
879 0 : show_smap_vma_flags(m, vma);
880 :
881 0 : return 0;
882 : }
883 :
884 0 : static int show_smaps_rollup(struct seq_file *m, void *v)
885 : {
886 0 : struct proc_maps_private *priv = m->private;
887 : struct mem_size_stats mss;
888 0 : struct mm_struct *mm = priv->mm;
889 : struct vm_area_struct *vma;
890 0 : unsigned long vma_start = 0, last_vma_end = 0;
891 0 : int ret = 0;
892 0 : VMA_ITERATOR(vmi, mm, 0);
893 :
894 0 : priv->task = get_proc_task(priv->inode);
895 0 : if (!priv->task)
896 : return -ESRCH;
897 :
898 0 : if (!mm || !mmget_not_zero(mm)) {
899 : ret = -ESRCH;
900 : goto out_put_task;
901 : }
902 :
903 0 : memset(&mss, 0, sizeof(mss));
904 :
905 0 : ret = mmap_read_lock_killable(mm);
906 0 : if (ret)
907 : goto out_put_mm;
908 :
909 0 : hold_task_mempolicy(priv);
910 0 : vma = vma_next(&vmi);
911 :
912 0 : if (unlikely(!vma))
913 : goto empty_set;
914 :
915 0 : vma_start = vma->vm_start;
916 : do {
917 0 : smap_gather_stats(vma, &mss, 0);
918 0 : last_vma_end = vma->vm_end;
919 :
920 : /*
921 : * Release mmap_lock temporarily if someone wants to
922 : * access it for write request.
923 : */
924 0 : if (mmap_lock_is_contended(mm)) {
925 0 : vma_iter_invalidate(&vmi);
926 0 : mmap_read_unlock(mm);
927 0 : ret = mmap_read_lock_killable(mm);
928 0 : if (ret) {
929 : release_task_mempolicy(priv);
930 : goto out_put_mm;
931 : }
932 :
933 : /*
934 : * After dropping the lock, there are four cases to
935 : * consider. See the following example for explanation.
936 : *
937 : * +------+------+-----------+
938 : * | VMA1 | VMA2 | VMA3 |
939 : * +------+------+-----------+
940 : * | | | |
941 : * 4k 8k 16k 400k
942 : *
943 : * Suppose we drop the lock after reading VMA2 due to
944 : * contention, then we get:
945 : *
946 : * last_vma_end = 16k
947 : *
948 : * 1) VMA2 is freed, but VMA3 exists:
949 : *
950 : * vma_next(vmi) will return VMA3.
951 : * In this case, just continue from VMA3.
952 : *
953 : * 2) VMA2 still exists:
954 : *
955 : * vma_next(vmi) will return VMA3.
956 : * In this case, just continue from VMA3.
957 : *
958 : * 3) No more VMAs can be found:
959 : *
960 : * vma_next(vmi) will return NULL.
961 : * No more things to do, just break.
962 : *
963 : * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
964 : *
965 : * vma_next(vmi) will return VMA' whose range
966 : * contains last_vma_end.
967 : * Iterate VMA' from last_vma_end.
968 : */
969 0 : vma = vma_next(&vmi);
970 : /* Case 3 above */
971 0 : if (!vma)
972 : break;
973 :
974 : /* Case 1 and 2 above */
975 0 : if (vma->vm_start >= last_vma_end)
976 0 : continue;
977 :
978 : /* Case 4 above */
979 0 : if (vma->vm_end > last_vma_end)
980 0 : smap_gather_stats(vma, &mss, last_vma_end);
981 : }
982 0 : } for_each_vma(vmi, vma);
983 :
984 : empty_set:
985 0 : show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0);
986 0 : seq_pad(m, ' ');
987 0 : seq_puts(m, "[rollup]\n");
988 :
989 0 : __show_smap(m, &mss, true);
990 :
991 0 : release_task_mempolicy(priv);
992 : mmap_read_unlock(mm);
993 :
994 : out_put_mm:
995 0 : mmput(mm);
996 : out_put_task:
997 0 : put_task_struct(priv->task);
998 0 : priv->task = NULL;
999 :
1000 0 : return ret;
1001 : }
1002 : #undef SEQ_PUT_DEC
1003 :
1004 : static const struct seq_operations proc_pid_smaps_op = {
1005 : .start = m_start,
1006 : .next = m_next,
1007 : .stop = m_stop,
1008 : .show = show_smap
1009 : };
1010 :
1011 0 : static int pid_smaps_open(struct inode *inode, struct file *file)
1012 : {
1013 0 : return do_maps_open(inode, file, &proc_pid_smaps_op);
1014 : }
1015 :
1016 0 : static int smaps_rollup_open(struct inode *inode, struct file *file)
1017 : {
1018 : int ret;
1019 : struct proc_maps_private *priv;
1020 :
1021 0 : priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
1022 0 : if (!priv)
1023 : return -ENOMEM;
1024 :
1025 0 : ret = single_open(file, show_smaps_rollup, priv);
1026 0 : if (ret)
1027 : goto out_free;
1028 :
1029 0 : priv->inode = inode;
1030 0 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
1031 0 : if (IS_ERR(priv->mm)) {
1032 0 : ret = PTR_ERR(priv->mm);
1033 :
1034 0 : single_release(inode, file);
1035 0 : goto out_free;
1036 : }
1037 :
1038 : return 0;
1039 :
1040 : out_free:
1041 0 : kfree(priv);
1042 0 : return ret;
1043 : }
1044 :
1045 0 : static int smaps_rollup_release(struct inode *inode, struct file *file)
1046 : {
1047 0 : struct seq_file *seq = file->private_data;
1048 0 : struct proc_maps_private *priv = seq->private;
1049 :
1050 0 : if (priv->mm)
1051 0 : mmdrop(priv->mm);
1052 :
1053 0 : kfree(priv);
1054 0 : return single_release(inode, file);
1055 : }
1056 :
1057 : const struct file_operations proc_pid_smaps_operations = {
1058 : .open = pid_smaps_open,
1059 : .read = seq_read,
1060 : .llseek = seq_lseek,
1061 : .release = proc_map_release,
1062 : };
1063 :
1064 : const struct file_operations proc_pid_smaps_rollup_operations = {
1065 : .open = smaps_rollup_open,
1066 : .read = seq_read,
1067 : .llseek = seq_lseek,
1068 : .release = smaps_rollup_release,
1069 : };
1070 :
1071 : enum clear_refs_types {
1072 : CLEAR_REFS_ALL = 1,
1073 : CLEAR_REFS_ANON,
1074 : CLEAR_REFS_MAPPED,
1075 : CLEAR_REFS_SOFT_DIRTY,
1076 : CLEAR_REFS_MM_HIWATER_RSS,
1077 : CLEAR_REFS_LAST,
1078 : };
1079 :
1080 : struct clear_refs_private {
1081 : enum clear_refs_types type;
1082 : };
1083 :
1084 : #ifdef CONFIG_MEM_SOFT_DIRTY
1085 :
1086 : static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1087 : {
1088 : struct page *page;
1089 :
1090 : if (!pte_write(pte))
1091 : return false;
1092 : if (!is_cow_mapping(vma->vm_flags))
1093 : return false;
1094 : if (likely(!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)))
1095 : return false;
1096 : page = vm_normal_page(vma, addr, pte);
1097 : if (!page)
1098 : return false;
1099 : return page_maybe_dma_pinned(page);
1100 : }
1101 :
1102 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1103 : unsigned long addr, pte_t *pte)
1104 : {
1105 : /*
1106 : * The soft-dirty tracker uses #PF-s to catch writes
1107 : * to pages, so write-protect the pte as well. See the
1108 : * Documentation/admin-guide/mm/soft-dirty.rst for full description
1109 : * of how soft-dirty works.
1110 : */
1111 : pte_t ptent = *pte;
1112 :
1113 : if (pte_present(ptent)) {
1114 : pte_t old_pte;
1115 :
1116 : if (pte_is_pinned(vma, addr, ptent))
1117 : return;
1118 : old_pte = ptep_modify_prot_start(vma, addr, pte);
1119 : ptent = pte_wrprotect(old_pte);
1120 : ptent = pte_clear_soft_dirty(ptent);
1121 : ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
1122 : } else if (is_swap_pte(ptent)) {
1123 : ptent = pte_swp_clear_soft_dirty(ptent);
1124 : set_pte_at(vma->vm_mm, addr, pte, ptent);
1125 : }
1126 : }
1127 : #else
1128 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1129 : unsigned long addr, pte_t *pte)
1130 : {
1131 : }
1132 : #endif
1133 :
1134 : #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1135 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1136 : unsigned long addr, pmd_t *pmdp)
1137 : {
1138 : pmd_t old, pmd = *pmdp;
1139 :
1140 : if (pmd_present(pmd)) {
1141 : /* See comment in change_huge_pmd() */
1142 : old = pmdp_invalidate(vma, addr, pmdp);
1143 : if (pmd_dirty(old))
1144 : pmd = pmd_mkdirty(pmd);
1145 : if (pmd_young(old))
1146 : pmd = pmd_mkyoung(pmd);
1147 :
1148 : pmd = pmd_wrprotect(pmd);
1149 : pmd = pmd_clear_soft_dirty(pmd);
1150 :
1151 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1152 : } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
1153 : pmd = pmd_swp_clear_soft_dirty(pmd);
1154 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1155 : }
1156 : }
1157 : #else
1158 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1159 : unsigned long addr, pmd_t *pmdp)
1160 : {
1161 : }
1162 : #endif
1163 :
1164 0 : static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
1165 : unsigned long end, struct mm_walk *walk)
1166 : {
1167 0 : struct clear_refs_private *cp = walk->private;
1168 0 : struct vm_area_struct *vma = walk->vma;
1169 : pte_t *pte, ptent;
1170 : spinlock_t *ptl;
1171 : struct page *page;
1172 :
1173 0 : ptl = pmd_trans_huge_lock(pmd, vma);
1174 : if (ptl) {
1175 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1176 : clear_soft_dirty_pmd(vma, addr, pmd);
1177 : goto out;
1178 : }
1179 :
1180 : if (!pmd_present(*pmd))
1181 : goto out;
1182 :
1183 : page = pmd_page(*pmd);
1184 :
1185 : /* Clear accessed and referenced bits. */
1186 : pmdp_test_and_clear_young(vma, addr, pmd);
1187 : test_and_clear_page_young(page);
1188 : ClearPageReferenced(page);
1189 : out:
1190 : spin_unlock(ptl);
1191 : return 0;
1192 : }
1193 :
1194 0 : if (pmd_trans_unstable(pmd))
1195 : return 0;
1196 :
1197 0 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1198 0 : for (; addr != end; pte++, addr += PAGE_SIZE) {
1199 0 : ptent = *pte;
1200 :
1201 0 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1202 0 : clear_soft_dirty(vma, addr, pte);
1203 0 : continue;
1204 : }
1205 :
1206 0 : if (!pte_present(ptent))
1207 0 : continue;
1208 :
1209 0 : page = vm_normal_page(vma, addr, ptent);
1210 0 : if (!page)
1211 0 : continue;
1212 :
1213 : /* Clear accessed and referenced bits. */
1214 0 : ptep_test_and_clear_young(vma, addr, pte);
1215 0 : test_and_clear_page_young(page);
1216 : ClearPageReferenced(page);
1217 : }
1218 0 : pte_unmap_unlock(pte - 1, ptl);
1219 0 : cond_resched();
1220 : return 0;
1221 : }
1222 :
1223 0 : static int clear_refs_test_walk(unsigned long start, unsigned long end,
1224 : struct mm_walk *walk)
1225 : {
1226 0 : struct clear_refs_private *cp = walk->private;
1227 0 : struct vm_area_struct *vma = walk->vma;
1228 :
1229 0 : if (vma->vm_flags & VM_PFNMAP)
1230 : return 1;
1231 :
1232 : /*
1233 : * Writing 1 to /proc/pid/clear_refs affects all pages.
1234 : * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1235 : * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1236 : * Writing 4 to /proc/pid/clear_refs affects all pages.
1237 : */
1238 0 : if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1239 : return 1;
1240 0 : if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1241 : return 1;
1242 0 : return 0;
1243 : }
1244 :
1245 : static const struct mm_walk_ops clear_refs_walk_ops = {
1246 : .pmd_entry = clear_refs_pte_range,
1247 : .test_walk = clear_refs_test_walk,
1248 : };
1249 :
1250 0 : static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1251 : size_t count, loff_t *ppos)
1252 : {
1253 : struct task_struct *task;
1254 : char buffer[PROC_NUMBUF];
1255 : struct mm_struct *mm;
1256 : struct vm_area_struct *vma;
1257 : enum clear_refs_types type;
1258 : int itype;
1259 : int rv;
1260 :
1261 0 : memset(buffer, 0, sizeof(buffer));
1262 0 : if (count > sizeof(buffer) - 1)
1263 0 : count = sizeof(buffer) - 1;
1264 0 : if (copy_from_user(buffer, buf, count))
1265 : return -EFAULT;
1266 0 : rv = kstrtoint(strstrip(buffer), 10, &itype);
1267 0 : if (rv < 0)
1268 0 : return rv;
1269 0 : type = (enum clear_refs_types)itype;
1270 0 : if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1271 : return -EINVAL;
1272 :
1273 0 : task = get_proc_task(file_inode(file));
1274 0 : if (!task)
1275 : return -ESRCH;
1276 0 : mm = get_task_mm(task);
1277 0 : if (mm) {
1278 0 : VMA_ITERATOR(vmi, mm, 0);
1279 : struct mmu_notifier_range range;
1280 0 : struct clear_refs_private cp = {
1281 : .type = type,
1282 : };
1283 :
1284 0 : if (mmap_write_lock_killable(mm)) {
1285 : count = -EINTR;
1286 : goto out_mm;
1287 : }
1288 0 : if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1289 : /*
1290 : * Writing 5 to /proc/pid/clear_refs resets the peak
1291 : * resident set size to this mm's current rss value.
1292 : */
1293 : reset_mm_hiwater_rss(mm);
1294 : goto out_unlock;
1295 : }
1296 :
1297 0 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1298 0 : for_each_vma(vmi, vma) {
1299 : if (!(vma->vm_flags & VM_SOFTDIRTY))
1300 0 : continue;
1301 : vm_flags_clear(vma, VM_SOFTDIRTY);
1302 : vma_set_page_prot(vma);
1303 : }
1304 :
1305 0 : inc_tlb_flush_pending(mm);
1306 : mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
1307 : 0, mm, 0, -1UL);
1308 : mmu_notifier_invalidate_range_start(&range);
1309 : }
1310 0 : walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp);
1311 0 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1312 0 : mmu_notifier_invalidate_range_end(&range);
1313 0 : flush_tlb_mm(mm);
1314 : dec_tlb_flush_pending(mm);
1315 : }
1316 : out_unlock:
1317 : mmap_write_unlock(mm);
1318 : out_mm:
1319 0 : mmput(mm);
1320 : }
1321 0 : put_task_struct(task);
1322 :
1323 0 : return count;
1324 : }
1325 :
1326 : const struct file_operations proc_clear_refs_operations = {
1327 : .write = clear_refs_write,
1328 : .llseek = noop_llseek,
1329 : };
1330 :
1331 : typedef struct {
1332 : u64 pme;
1333 : } pagemap_entry_t;
1334 :
1335 : struct pagemapread {
1336 : int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
1337 : pagemap_entry_t *buffer;
1338 : bool show_pfn;
1339 : };
1340 :
1341 : #define PAGEMAP_WALK_SIZE (PMD_SIZE)
1342 : #define PAGEMAP_WALK_MASK (PMD_MASK)
1343 :
1344 : #define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1345 : #define PM_PFRAME_BITS 55
1346 : #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1347 : #define PM_SOFT_DIRTY BIT_ULL(55)
1348 : #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1349 : #define PM_UFFD_WP BIT_ULL(57)
1350 : #define PM_FILE BIT_ULL(61)
1351 : #define PM_SWAP BIT_ULL(62)
1352 : #define PM_PRESENT BIT_ULL(63)
1353 :
1354 : #define PM_END_OF_BUFFER 1
1355 :
1356 : static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1357 : {
1358 0 : return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1359 : }
1360 :
1361 : static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1362 : struct pagemapread *pm)
1363 : {
1364 0 : pm->buffer[pm->pos++] = *pme;
1365 0 : if (pm->pos >= pm->len)
1366 : return PM_END_OF_BUFFER;
1367 : return 0;
1368 : }
1369 :
1370 0 : static int pagemap_pte_hole(unsigned long start, unsigned long end,
1371 : __always_unused int depth, struct mm_walk *walk)
1372 : {
1373 0 : struct pagemapread *pm = walk->private;
1374 0 : unsigned long addr = start;
1375 0 : int err = 0;
1376 :
1377 0 : while (addr < end) {
1378 0 : struct vm_area_struct *vma = find_vma(walk->mm, addr);
1379 0 : pagemap_entry_t pme = make_pme(0, 0);
1380 : /* End of address space hole, which we mark as non-present. */
1381 : unsigned long hole_end;
1382 :
1383 0 : if (vma)
1384 0 : hole_end = min(end, vma->vm_start);
1385 : else
1386 : hole_end = end;
1387 :
1388 0 : for (; addr < hole_end; addr += PAGE_SIZE) {
1389 0 : err = add_to_pagemap(addr, &pme, pm);
1390 0 : if (err)
1391 : goto out;
1392 : }
1393 :
1394 0 : if (!vma)
1395 : break;
1396 :
1397 : /* Addresses in the VMA. */
1398 : if (vma->vm_flags & VM_SOFTDIRTY)
1399 : pme = make_pme(0, PM_SOFT_DIRTY);
1400 0 : for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1401 0 : err = add_to_pagemap(addr, &pme, pm);
1402 0 : if (err)
1403 : goto out;
1404 : }
1405 : }
1406 : out:
1407 0 : return err;
1408 : }
1409 :
1410 0 : static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1411 : struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1412 : {
1413 0 : u64 frame = 0, flags = 0;
1414 0 : struct page *page = NULL;
1415 0 : bool migration = false;
1416 :
1417 0 : if (pte_present(pte)) {
1418 0 : if (pm->show_pfn)
1419 0 : frame = pte_pfn(pte);
1420 0 : flags |= PM_PRESENT;
1421 0 : page = vm_normal_page(vma, addr, pte);
1422 0 : if (pte_soft_dirty(pte))
1423 : flags |= PM_SOFT_DIRTY;
1424 : if (pte_uffd_wp(pte))
1425 : flags |= PM_UFFD_WP;
1426 0 : } else if (is_swap_pte(pte)) {
1427 : swp_entry_t entry;
1428 0 : if (pte_swp_soft_dirty(pte))
1429 : flags |= PM_SOFT_DIRTY;
1430 : if (pte_swp_uffd_wp(pte))
1431 : flags |= PM_UFFD_WP;
1432 0 : entry = pte_to_swp_entry(pte);
1433 0 : if (pm->show_pfn) {
1434 : pgoff_t offset;
1435 : /*
1436 : * For PFN swap offsets, keeping the offset field
1437 : * to be PFN only to be compatible with old smaps.
1438 : */
1439 0 : if (is_pfn_swap_entry(entry))
1440 0 : offset = swp_offset_pfn(entry);
1441 : else
1442 0 : offset = swp_offset(entry);
1443 0 : frame = swp_type(entry) |
1444 0 : (offset << MAX_SWAPFILES_SHIFT);
1445 : }
1446 0 : flags |= PM_SWAP;
1447 0 : migration = is_migration_entry(entry);
1448 0 : if (is_pfn_swap_entry(entry))
1449 0 : page = pfn_swap_entry_to_page(entry);
1450 : if (pte_marker_entry_uffd_wp(entry))
1451 : flags |= PM_UFFD_WP;
1452 : }
1453 :
1454 0 : if (page && !PageAnon(page))
1455 0 : flags |= PM_FILE;
1456 0 : if (page && !migration && page_mapcount(page) == 1)
1457 0 : flags |= PM_MMAP_EXCLUSIVE;
1458 : if (vma->vm_flags & VM_SOFTDIRTY)
1459 : flags |= PM_SOFT_DIRTY;
1460 :
1461 0 : return make_pme(frame, flags);
1462 : }
1463 :
1464 0 : static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1465 : struct mm_walk *walk)
1466 : {
1467 0 : struct vm_area_struct *vma = walk->vma;
1468 0 : struct pagemapread *pm = walk->private;
1469 : spinlock_t *ptl;
1470 : pte_t *pte, *orig_pte;
1471 0 : int err = 0;
1472 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1473 : bool migration = false;
1474 :
1475 : ptl = pmd_trans_huge_lock(pmdp, vma);
1476 : if (ptl) {
1477 : u64 flags = 0, frame = 0;
1478 : pmd_t pmd = *pmdp;
1479 : struct page *page = NULL;
1480 :
1481 : if (vma->vm_flags & VM_SOFTDIRTY)
1482 : flags |= PM_SOFT_DIRTY;
1483 :
1484 : if (pmd_present(pmd)) {
1485 : page = pmd_page(pmd);
1486 :
1487 : flags |= PM_PRESENT;
1488 : if (pmd_soft_dirty(pmd))
1489 : flags |= PM_SOFT_DIRTY;
1490 : if (pmd_uffd_wp(pmd))
1491 : flags |= PM_UFFD_WP;
1492 : if (pm->show_pfn)
1493 : frame = pmd_pfn(pmd) +
1494 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1495 : }
1496 : #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1497 : else if (is_swap_pmd(pmd)) {
1498 : swp_entry_t entry = pmd_to_swp_entry(pmd);
1499 : unsigned long offset;
1500 :
1501 : if (pm->show_pfn) {
1502 : if (is_pfn_swap_entry(entry))
1503 : offset = swp_offset_pfn(entry);
1504 : else
1505 : offset = swp_offset(entry);
1506 : offset = offset +
1507 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1508 : frame = swp_type(entry) |
1509 : (offset << MAX_SWAPFILES_SHIFT);
1510 : }
1511 : flags |= PM_SWAP;
1512 : if (pmd_swp_soft_dirty(pmd))
1513 : flags |= PM_SOFT_DIRTY;
1514 : if (pmd_swp_uffd_wp(pmd))
1515 : flags |= PM_UFFD_WP;
1516 : VM_BUG_ON(!is_pmd_migration_entry(pmd));
1517 : migration = is_migration_entry(entry);
1518 : page = pfn_swap_entry_to_page(entry);
1519 : }
1520 : #endif
1521 :
1522 : if (page && !migration && page_mapcount(page) == 1)
1523 : flags |= PM_MMAP_EXCLUSIVE;
1524 :
1525 : for (; addr != end; addr += PAGE_SIZE) {
1526 : pagemap_entry_t pme = make_pme(frame, flags);
1527 :
1528 : err = add_to_pagemap(addr, &pme, pm);
1529 : if (err)
1530 : break;
1531 : if (pm->show_pfn) {
1532 : if (flags & PM_PRESENT)
1533 : frame++;
1534 : else if (flags & PM_SWAP)
1535 : frame += (1 << MAX_SWAPFILES_SHIFT);
1536 : }
1537 : }
1538 : spin_unlock(ptl);
1539 : return err;
1540 : }
1541 :
1542 : if (pmd_trans_unstable(pmdp))
1543 : return 0;
1544 : #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1545 :
1546 : /*
1547 : * We can assume that @vma always points to a valid one and @end never
1548 : * goes beyond vma->vm_end.
1549 : */
1550 0 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1551 0 : for (; addr < end; pte++, addr += PAGE_SIZE) {
1552 : pagemap_entry_t pme;
1553 :
1554 0 : pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1555 0 : err = add_to_pagemap(addr, &pme, pm);
1556 0 : if (err)
1557 : break;
1558 : }
1559 0 : pte_unmap_unlock(orig_pte, ptl);
1560 :
1561 0 : cond_resched();
1562 :
1563 0 : return err;
1564 : }
1565 :
1566 : #ifdef CONFIG_HUGETLB_PAGE
1567 : /* This function walks within one hugetlb entry in the single call */
1568 : static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1569 : unsigned long addr, unsigned long end,
1570 : struct mm_walk *walk)
1571 : {
1572 : struct pagemapread *pm = walk->private;
1573 : struct vm_area_struct *vma = walk->vma;
1574 : u64 flags = 0, frame = 0;
1575 : int err = 0;
1576 : pte_t pte;
1577 :
1578 : if (vma->vm_flags & VM_SOFTDIRTY)
1579 : flags |= PM_SOFT_DIRTY;
1580 :
1581 : pte = huge_ptep_get(ptep);
1582 : if (pte_present(pte)) {
1583 : struct page *page = pte_page(pte);
1584 :
1585 : if (!PageAnon(page))
1586 : flags |= PM_FILE;
1587 :
1588 : if (page_mapcount(page) == 1)
1589 : flags |= PM_MMAP_EXCLUSIVE;
1590 :
1591 : if (huge_pte_uffd_wp(pte))
1592 : flags |= PM_UFFD_WP;
1593 :
1594 : flags |= PM_PRESENT;
1595 : if (pm->show_pfn)
1596 : frame = pte_pfn(pte) +
1597 : ((addr & ~hmask) >> PAGE_SHIFT);
1598 : } else if (pte_swp_uffd_wp_any(pte)) {
1599 : flags |= PM_UFFD_WP;
1600 : }
1601 :
1602 : for (; addr != end; addr += PAGE_SIZE) {
1603 : pagemap_entry_t pme = make_pme(frame, flags);
1604 :
1605 : err = add_to_pagemap(addr, &pme, pm);
1606 : if (err)
1607 : return err;
1608 : if (pm->show_pfn && (flags & PM_PRESENT))
1609 : frame++;
1610 : }
1611 :
1612 : cond_resched();
1613 :
1614 : return err;
1615 : }
1616 : #else
1617 : #define pagemap_hugetlb_range NULL
1618 : #endif /* HUGETLB_PAGE */
1619 :
1620 : static const struct mm_walk_ops pagemap_ops = {
1621 : .pmd_entry = pagemap_pmd_range,
1622 : .pte_hole = pagemap_pte_hole,
1623 : .hugetlb_entry = pagemap_hugetlb_range,
1624 : };
1625 :
1626 : /*
1627 : * /proc/pid/pagemap - an array mapping virtual pages to pfns
1628 : *
1629 : * For each page in the address space, this file contains one 64-bit entry
1630 : * consisting of the following:
1631 : *
1632 : * Bits 0-54 page frame number (PFN) if present
1633 : * Bits 0-4 swap type if swapped
1634 : * Bits 5-54 swap offset if swapped
1635 : * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1636 : * Bit 56 page exclusively mapped
1637 : * Bit 57 pte is uffd-wp write-protected
1638 : * Bits 58-60 zero
1639 : * Bit 61 page is file-page or shared-anon
1640 : * Bit 62 page swapped
1641 : * Bit 63 page present
1642 : *
1643 : * If the page is not present but in swap, then the PFN contains an
1644 : * encoding of the swap file number and the page's offset into the
1645 : * swap. Unmapped pages return a null PFN. This allows determining
1646 : * precisely which pages are mapped (or in swap) and comparing mapped
1647 : * pages between processes.
1648 : *
1649 : * Efficient users of this interface will use /proc/pid/maps to
1650 : * determine which areas of memory are actually mapped and llseek to
1651 : * skip over unmapped regions.
1652 : */
1653 0 : static ssize_t pagemap_read(struct file *file, char __user *buf,
1654 : size_t count, loff_t *ppos)
1655 : {
1656 0 : struct mm_struct *mm = file->private_data;
1657 : struct pagemapread pm;
1658 : unsigned long src;
1659 : unsigned long svpfn;
1660 : unsigned long start_vaddr;
1661 : unsigned long end_vaddr;
1662 0 : int ret = 0, copied = 0;
1663 :
1664 0 : if (!mm || !mmget_not_zero(mm))
1665 : goto out;
1666 :
1667 0 : ret = -EINVAL;
1668 : /* file position must be aligned */
1669 0 : if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1670 : goto out_mm;
1671 :
1672 0 : ret = 0;
1673 0 : if (!count)
1674 : goto out_mm;
1675 :
1676 : /* do not disclose physical addresses: attack vector */
1677 0 : pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1678 :
1679 0 : pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1680 0 : pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1681 0 : ret = -ENOMEM;
1682 0 : if (!pm.buffer)
1683 : goto out_mm;
1684 :
1685 0 : src = *ppos;
1686 0 : svpfn = src / PM_ENTRY_BYTES;
1687 0 : end_vaddr = mm->task_size;
1688 :
1689 : /* watch out for wraparound */
1690 0 : start_vaddr = end_vaddr;
1691 0 : if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) {
1692 0 : ret = mmap_read_lock_killable(mm);
1693 0 : if (ret)
1694 : goto out_free;
1695 0 : start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT);
1696 : mmap_read_unlock(mm);
1697 : }
1698 :
1699 : /* Ensure the address is inside the task */
1700 0 : if (start_vaddr > mm->task_size)
1701 0 : start_vaddr = end_vaddr;
1702 :
1703 : /*
1704 : * The odds are that this will stop walking way
1705 : * before end_vaddr, because the length of the
1706 : * user buffer is tracked in "pm", and the walk
1707 : * will stop when we hit the end of the buffer.
1708 : */
1709 : ret = 0;
1710 0 : while (count && (start_vaddr < end_vaddr)) {
1711 : int len;
1712 : unsigned long end;
1713 :
1714 0 : pm.pos = 0;
1715 0 : end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1716 : /* overflow ? */
1717 0 : if (end < start_vaddr || end > end_vaddr)
1718 0 : end = end_vaddr;
1719 0 : ret = mmap_read_lock_killable(mm);
1720 0 : if (ret)
1721 : goto out_free;
1722 0 : ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm);
1723 0 : mmap_read_unlock(mm);
1724 0 : start_vaddr = end;
1725 :
1726 0 : len = min(count, PM_ENTRY_BYTES * pm.pos);
1727 0 : if (copy_to_user(buf, pm.buffer, len)) {
1728 : ret = -EFAULT;
1729 : goto out_free;
1730 : }
1731 0 : copied += len;
1732 0 : buf += len;
1733 0 : count -= len;
1734 : }
1735 0 : *ppos += copied;
1736 0 : if (!ret || ret == PM_END_OF_BUFFER)
1737 0 : ret = copied;
1738 :
1739 : out_free:
1740 0 : kfree(pm.buffer);
1741 : out_mm:
1742 0 : mmput(mm);
1743 : out:
1744 0 : return ret;
1745 : }
1746 :
1747 0 : static int pagemap_open(struct inode *inode, struct file *file)
1748 : {
1749 : struct mm_struct *mm;
1750 :
1751 0 : mm = proc_mem_open(inode, PTRACE_MODE_READ);
1752 0 : if (IS_ERR(mm))
1753 0 : return PTR_ERR(mm);
1754 0 : file->private_data = mm;
1755 0 : return 0;
1756 : }
1757 :
1758 0 : static int pagemap_release(struct inode *inode, struct file *file)
1759 : {
1760 0 : struct mm_struct *mm = file->private_data;
1761 :
1762 0 : if (mm)
1763 : mmdrop(mm);
1764 0 : return 0;
1765 : }
1766 :
1767 : const struct file_operations proc_pagemap_operations = {
1768 : .llseek = mem_lseek, /* borrow this */
1769 : .read = pagemap_read,
1770 : .open = pagemap_open,
1771 : .release = pagemap_release,
1772 : };
1773 : #endif /* CONFIG_PROC_PAGE_MONITOR */
1774 :
1775 : #ifdef CONFIG_NUMA
1776 :
1777 : struct numa_maps {
1778 : unsigned long pages;
1779 : unsigned long anon;
1780 : unsigned long active;
1781 : unsigned long writeback;
1782 : unsigned long mapcount_max;
1783 : unsigned long dirty;
1784 : unsigned long swapcache;
1785 : unsigned long node[MAX_NUMNODES];
1786 : };
1787 :
1788 : struct numa_maps_private {
1789 : struct proc_maps_private proc_maps;
1790 : struct numa_maps md;
1791 : };
1792 :
1793 : static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1794 : unsigned long nr_pages)
1795 : {
1796 : int count = page_mapcount(page);
1797 :
1798 : md->pages += nr_pages;
1799 : if (pte_dirty || PageDirty(page))
1800 : md->dirty += nr_pages;
1801 :
1802 : if (PageSwapCache(page))
1803 : md->swapcache += nr_pages;
1804 :
1805 : if (PageActive(page) || PageUnevictable(page))
1806 : md->active += nr_pages;
1807 :
1808 : if (PageWriteback(page))
1809 : md->writeback += nr_pages;
1810 :
1811 : if (PageAnon(page))
1812 : md->anon += nr_pages;
1813 :
1814 : if (count > md->mapcount_max)
1815 : md->mapcount_max = count;
1816 :
1817 : md->node[page_to_nid(page)] += nr_pages;
1818 : }
1819 :
1820 : static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1821 : unsigned long addr)
1822 : {
1823 : struct page *page;
1824 : int nid;
1825 :
1826 : if (!pte_present(pte))
1827 : return NULL;
1828 :
1829 : page = vm_normal_page(vma, addr, pte);
1830 : if (!page || is_zone_device_page(page))
1831 : return NULL;
1832 :
1833 : if (PageReserved(page))
1834 : return NULL;
1835 :
1836 : nid = page_to_nid(page);
1837 : if (!node_isset(nid, node_states[N_MEMORY]))
1838 : return NULL;
1839 :
1840 : return page;
1841 : }
1842 :
1843 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1844 : static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1845 : struct vm_area_struct *vma,
1846 : unsigned long addr)
1847 : {
1848 : struct page *page;
1849 : int nid;
1850 :
1851 : if (!pmd_present(pmd))
1852 : return NULL;
1853 :
1854 : page = vm_normal_page_pmd(vma, addr, pmd);
1855 : if (!page)
1856 : return NULL;
1857 :
1858 : if (PageReserved(page))
1859 : return NULL;
1860 :
1861 : nid = page_to_nid(page);
1862 : if (!node_isset(nid, node_states[N_MEMORY]))
1863 : return NULL;
1864 :
1865 : return page;
1866 : }
1867 : #endif
1868 :
1869 : static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1870 : unsigned long end, struct mm_walk *walk)
1871 : {
1872 : struct numa_maps *md = walk->private;
1873 : struct vm_area_struct *vma = walk->vma;
1874 : spinlock_t *ptl;
1875 : pte_t *orig_pte;
1876 : pte_t *pte;
1877 :
1878 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1879 : ptl = pmd_trans_huge_lock(pmd, vma);
1880 : if (ptl) {
1881 : struct page *page;
1882 :
1883 : page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1884 : if (page)
1885 : gather_stats(page, md, pmd_dirty(*pmd),
1886 : HPAGE_PMD_SIZE/PAGE_SIZE);
1887 : spin_unlock(ptl);
1888 : return 0;
1889 : }
1890 :
1891 : if (pmd_trans_unstable(pmd))
1892 : return 0;
1893 : #endif
1894 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1895 : do {
1896 : struct page *page = can_gather_numa_stats(*pte, vma, addr);
1897 : if (!page)
1898 : continue;
1899 : gather_stats(page, md, pte_dirty(*pte), 1);
1900 :
1901 : } while (pte++, addr += PAGE_SIZE, addr != end);
1902 : pte_unmap_unlock(orig_pte, ptl);
1903 : cond_resched();
1904 : return 0;
1905 : }
1906 : #ifdef CONFIG_HUGETLB_PAGE
1907 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1908 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1909 : {
1910 : pte_t huge_pte = huge_ptep_get(pte);
1911 : struct numa_maps *md;
1912 : struct page *page;
1913 :
1914 : if (!pte_present(huge_pte))
1915 : return 0;
1916 :
1917 : page = pte_page(huge_pte);
1918 :
1919 : md = walk->private;
1920 : gather_stats(page, md, pte_dirty(huge_pte), 1);
1921 : return 0;
1922 : }
1923 :
1924 : #else
1925 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1926 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1927 : {
1928 : return 0;
1929 : }
1930 : #endif
1931 :
1932 : static const struct mm_walk_ops show_numa_ops = {
1933 : .hugetlb_entry = gather_hugetlb_stats,
1934 : .pmd_entry = gather_pte_stats,
1935 : };
1936 :
1937 : /*
1938 : * Display pages allocated per node and memory policy via /proc.
1939 : */
1940 : static int show_numa_map(struct seq_file *m, void *v)
1941 : {
1942 : struct numa_maps_private *numa_priv = m->private;
1943 : struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1944 : struct vm_area_struct *vma = v;
1945 : struct numa_maps *md = &numa_priv->md;
1946 : struct file *file = vma->vm_file;
1947 : struct mm_struct *mm = vma->vm_mm;
1948 : struct mempolicy *pol;
1949 : char buffer[64];
1950 : int nid;
1951 :
1952 : if (!mm)
1953 : return 0;
1954 :
1955 : /* Ensure we start with an empty set of numa_maps statistics. */
1956 : memset(md, 0, sizeof(*md));
1957 :
1958 : pol = __get_vma_policy(vma, vma->vm_start);
1959 : if (pol) {
1960 : mpol_to_str(buffer, sizeof(buffer), pol);
1961 : mpol_cond_put(pol);
1962 : } else {
1963 : mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1964 : }
1965 :
1966 : seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1967 :
1968 : if (file) {
1969 : seq_puts(m, " file=");
1970 : seq_file_path(m, file, "\n\t= ");
1971 : } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1972 : seq_puts(m, " heap");
1973 : } else if (is_stack(vma)) {
1974 : seq_puts(m, " stack");
1975 : }
1976 :
1977 : if (is_vm_hugetlb_page(vma))
1978 : seq_puts(m, " huge");
1979 :
1980 : /* mmap_lock is held by m_start */
1981 : walk_page_vma(vma, &show_numa_ops, md);
1982 :
1983 : if (!md->pages)
1984 : goto out;
1985 :
1986 : if (md->anon)
1987 : seq_printf(m, " anon=%lu", md->anon);
1988 :
1989 : if (md->dirty)
1990 : seq_printf(m, " dirty=%lu", md->dirty);
1991 :
1992 : if (md->pages != md->anon && md->pages != md->dirty)
1993 : seq_printf(m, " mapped=%lu", md->pages);
1994 :
1995 : if (md->mapcount_max > 1)
1996 : seq_printf(m, " mapmax=%lu", md->mapcount_max);
1997 :
1998 : if (md->swapcache)
1999 : seq_printf(m, " swapcache=%lu", md->swapcache);
2000 :
2001 : if (md->active < md->pages && !is_vm_hugetlb_page(vma))
2002 : seq_printf(m, " active=%lu", md->active);
2003 :
2004 : if (md->writeback)
2005 : seq_printf(m, " writeback=%lu", md->writeback);
2006 :
2007 : for_each_node_state(nid, N_MEMORY)
2008 : if (md->node[nid])
2009 : seq_printf(m, " N%d=%lu", nid, md->node[nid]);
2010 :
2011 : seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
2012 : out:
2013 : seq_putc(m, '\n');
2014 : return 0;
2015 : }
2016 :
2017 : static const struct seq_operations proc_pid_numa_maps_op = {
2018 : .start = m_start,
2019 : .next = m_next,
2020 : .stop = m_stop,
2021 : .show = show_numa_map,
2022 : };
2023 :
2024 : static int pid_numa_maps_open(struct inode *inode, struct file *file)
2025 : {
2026 : return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
2027 : sizeof(struct numa_maps_private));
2028 : }
2029 :
2030 : const struct file_operations proc_pid_numa_maps_operations = {
2031 : .open = pid_numa_maps_open,
2032 : .read = seq_read,
2033 : .llseek = seq_lseek,
2034 : .release = proc_map_release,
2035 : };
2036 :
2037 : #endif /* CONFIG_NUMA */
|
