Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
4 : *
5 : * Swap reorganised 29.12.95, Stephen Tweedie.
6 : * kswapd added: 7.1.96 sct
7 : * Removed kswapd_ctl limits, and swap out as many pages as needed
8 : * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
9 : * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
10 : * Multiqueue VM started 5.8.00, Rik van Riel.
11 : */
12 :
13 : #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 :
15 : #include <linux/mm.h>
16 : #include <linux/sched/mm.h>
17 : #include <linux/module.h>
18 : #include <linux/gfp.h>
19 : #include <linux/kernel_stat.h>
20 : #include <linux/swap.h>
21 : #include <linux/pagemap.h>
22 : #include <linux/init.h>
23 : #include <linux/highmem.h>
24 : #include <linux/vmpressure.h>
25 : #include <linux/vmstat.h>
26 : #include <linux/file.h>
27 : #include <linux/writeback.h>
28 : #include <linux/blkdev.h>
29 : #include <linux/buffer_head.h> /* for buffer_heads_over_limit */
30 : #include <linux/mm_inline.h>
31 : #include <linux/backing-dev.h>
32 : #include <linux/rmap.h>
33 : #include <linux/topology.h>
34 : #include <linux/cpu.h>
35 : #include <linux/cpuset.h>
36 : #include <linux/compaction.h>
37 : #include <linux/notifier.h>
38 : #include <linux/mutex.h>
39 : #include <linux/delay.h>
40 : #include <linux/kthread.h>
41 : #include <linux/freezer.h>
42 : #include <linux/memcontrol.h>
43 : #include <linux/migrate.h>
44 : #include <linux/delayacct.h>
45 : #include <linux/sysctl.h>
46 : #include <linux/memory-tiers.h>
47 : #include <linux/oom.h>
48 : #include <linux/pagevec.h>
49 : #include <linux/prefetch.h>
50 : #include <linux/printk.h>
51 : #include <linux/dax.h>
52 : #include <linux/psi.h>
53 : #include <linux/pagewalk.h>
54 : #include <linux/shmem_fs.h>
55 : #include <linux/ctype.h>
56 : #include <linux/debugfs.h>
57 : #include <linux/khugepaged.h>
58 : #include <linux/rculist_nulls.h>
59 : #include <linux/random.h>
60 : #include <linux/srcu.h>
61 :
62 : #include <asm/tlbflush.h>
63 : #include <asm/div64.h>
64 :
65 : #include <linux/swapops.h>
66 : #include <linux/balloon_compaction.h>
67 : #include <linux/sched/sysctl.h>
68 :
69 : #include "internal.h"
70 : #include "swap.h"
71 :
72 : #define CREATE_TRACE_POINTS
73 : #include <trace/events/vmscan.h>
74 :
75 : struct scan_control {
76 : /* How many pages shrink_list() should reclaim */
77 : unsigned long nr_to_reclaim;
78 :
79 : /*
80 : * Nodemask of nodes allowed by the caller. If NULL, all nodes
81 : * are scanned.
82 : */
83 : nodemask_t *nodemask;
84 :
85 : /*
86 : * The memory cgroup that hit its limit and as a result is the
87 : * primary target of this reclaim invocation.
88 : */
89 : struct mem_cgroup *target_mem_cgroup;
90 :
91 : /*
92 : * Scan pressure balancing between anon and file LRUs
93 : */
94 : unsigned long anon_cost;
95 : unsigned long file_cost;
96 :
97 : /* Can active folios be deactivated as part of reclaim? */
98 : #define DEACTIVATE_ANON 1
99 : #define DEACTIVATE_FILE 2
100 : unsigned int may_deactivate:2;
101 : unsigned int force_deactivate:1;
102 : unsigned int skipped_deactivate:1;
103 :
104 : /* Writepage batching in laptop mode; RECLAIM_WRITE */
105 : unsigned int may_writepage:1;
106 :
107 : /* Can mapped folios be reclaimed? */
108 : unsigned int may_unmap:1;
109 :
110 : /* Can folios be swapped as part of reclaim? */
111 : unsigned int may_swap:1;
112 :
113 : /* Proactive reclaim invoked by userspace through memory.reclaim */
114 : unsigned int proactive:1;
115 :
116 : /*
117 : * Cgroup memory below memory.low is protected as long as we
118 : * don't threaten to OOM. If any cgroup is reclaimed at
119 : * reduced force or passed over entirely due to its memory.low
120 : * setting (memcg_low_skipped), and nothing is reclaimed as a
121 : * result, then go back for one more cycle that reclaims the protected
122 : * memory (memcg_low_reclaim) to avert OOM.
123 : */
124 : unsigned int memcg_low_reclaim:1;
125 : unsigned int memcg_low_skipped:1;
126 :
127 : unsigned int hibernation_mode:1;
128 :
129 : /* One of the zones is ready for compaction */
130 : unsigned int compaction_ready:1;
131 :
132 : /* There is easily reclaimable cold cache in the current node */
133 : unsigned int cache_trim_mode:1;
134 :
135 : /* The file folios on the current node are dangerously low */
136 : unsigned int file_is_tiny:1;
137 :
138 : /* Always discard instead of demoting to lower tier memory */
139 : unsigned int no_demotion:1;
140 :
141 : /* Allocation order */
142 : s8 order;
143 :
144 : /* Scan (total_size >> priority) pages at once */
145 : s8 priority;
146 :
147 : /* The highest zone to isolate folios for reclaim from */
148 : s8 reclaim_idx;
149 :
150 : /* This context's GFP mask */
151 : gfp_t gfp_mask;
152 :
153 : /* Incremented by the number of inactive pages that were scanned */
154 : unsigned long nr_scanned;
155 :
156 : /* Number of pages freed so far during a call to shrink_zones() */
157 : unsigned long nr_reclaimed;
158 :
159 : struct {
160 : unsigned int dirty;
161 : unsigned int unqueued_dirty;
162 : unsigned int congested;
163 : unsigned int writeback;
164 : unsigned int immediate;
165 : unsigned int file_taken;
166 : unsigned int taken;
167 : } nr;
168 :
169 : /* for recording the reclaimed slab by now */
170 : struct reclaim_state reclaim_state;
171 : };
172 :
173 : #ifdef ARCH_HAS_PREFETCHW
174 : #define prefetchw_prev_lru_folio(_folio, _base, _field) \
175 : do { \
176 : if ((_folio)->lru.prev != _base) { \
177 : struct folio *prev; \
178 : \
179 : prev = lru_to_folio(&(_folio->lru)); \
180 : prefetchw(&prev->_field); \
181 : } \
182 : } while (0)
183 : #else
184 : #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0)
185 : #endif
186 :
187 : /*
188 : * From 0 .. 200. Higher means more swappy.
189 : */
190 : int vm_swappiness = 60;
191 :
192 : LIST_HEAD(shrinker_list);
193 : DEFINE_MUTEX(shrinker_mutex);
194 : DEFINE_SRCU(shrinker_srcu);
195 : static atomic_t shrinker_srcu_generation = ATOMIC_INIT(0);
196 :
197 : #ifdef CONFIG_MEMCG
198 : static int shrinker_nr_max;
199 :
200 : /* The shrinker_info is expanded in a batch of BITS_PER_LONG */
201 : static inline int shrinker_map_size(int nr_items)
202 : {
203 : return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long));
204 : }
205 :
206 : static inline int shrinker_defer_size(int nr_items)
207 : {
208 : return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t));
209 : }
210 :
211 : static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
212 : int nid)
213 : {
214 : return srcu_dereference_check(memcg->nodeinfo[nid]->shrinker_info,
215 : &shrinker_srcu,
216 : lockdep_is_held(&shrinker_mutex));
217 : }
218 :
219 : static struct shrinker_info *shrinker_info_srcu(struct mem_cgroup *memcg,
220 : int nid)
221 : {
222 : return srcu_dereference(memcg->nodeinfo[nid]->shrinker_info,
223 : &shrinker_srcu);
224 : }
225 :
226 : static void free_shrinker_info_rcu(struct rcu_head *head)
227 : {
228 : kvfree(container_of(head, struct shrinker_info, rcu));
229 : }
230 :
231 : static int expand_one_shrinker_info(struct mem_cgroup *memcg,
232 : int map_size, int defer_size,
233 : int old_map_size, int old_defer_size,
234 : int new_nr_max)
235 : {
236 : struct shrinker_info *new, *old;
237 : struct mem_cgroup_per_node *pn;
238 : int nid;
239 : int size = map_size + defer_size;
240 :
241 : for_each_node(nid) {
242 : pn = memcg->nodeinfo[nid];
243 : old = shrinker_info_protected(memcg, nid);
244 : /* Not yet online memcg */
245 : if (!old)
246 : return 0;
247 :
248 : /* Already expanded this shrinker_info */
249 : if (new_nr_max <= old->map_nr_max)
250 : continue;
251 :
252 : new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
253 : if (!new)
254 : return -ENOMEM;
255 :
256 : new->nr_deferred = (atomic_long_t *)(new + 1);
257 : new->map = (void *)new->nr_deferred + defer_size;
258 : new->map_nr_max = new_nr_max;
259 :
260 : /* map: set all old bits, clear all new bits */
261 : memset(new->map, (int)0xff, old_map_size);
262 : memset((void *)new->map + old_map_size, 0, map_size - old_map_size);
263 : /* nr_deferred: copy old values, clear all new values */
264 : memcpy(new->nr_deferred, old->nr_deferred, old_defer_size);
265 : memset((void *)new->nr_deferred + old_defer_size, 0,
266 : defer_size - old_defer_size);
267 :
268 : rcu_assign_pointer(pn->shrinker_info, new);
269 : call_srcu(&shrinker_srcu, &old->rcu, free_shrinker_info_rcu);
270 : }
271 :
272 : return 0;
273 : }
274 :
275 : void free_shrinker_info(struct mem_cgroup *memcg)
276 : {
277 : struct mem_cgroup_per_node *pn;
278 : struct shrinker_info *info;
279 : int nid;
280 :
281 : for_each_node(nid) {
282 : pn = memcg->nodeinfo[nid];
283 : info = rcu_dereference_protected(pn->shrinker_info, true);
284 : kvfree(info);
285 : rcu_assign_pointer(pn->shrinker_info, NULL);
286 : }
287 : }
288 :
289 : int alloc_shrinker_info(struct mem_cgroup *memcg)
290 : {
291 : struct shrinker_info *info;
292 : int nid, size, ret = 0;
293 : int map_size, defer_size = 0;
294 :
295 : mutex_lock(&shrinker_mutex);
296 : map_size = shrinker_map_size(shrinker_nr_max);
297 : defer_size = shrinker_defer_size(shrinker_nr_max);
298 : size = map_size + defer_size;
299 : for_each_node(nid) {
300 : info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid);
301 : if (!info) {
302 : free_shrinker_info(memcg);
303 : ret = -ENOMEM;
304 : break;
305 : }
306 : info->nr_deferred = (atomic_long_t *)(info + 1);
307 : info->map = (void *)info->nr_deferred + defer_size;
308 : info->map_nr_max = shrinker_nr_max;
309 : rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
310 : }
311 : mutex_unlock(&shrinker_mutex);
312 :
313 : return ret;
314 : }
315 :
316 : static int expand_shrinker_info(int new_id)
317 : {
318 : int ret = 0;
319 : int new_nr_max = round_up(new_id + 1, BITS_PER_LONG);
320 : int map_size, defer_size = 0;
321 : int old_map_size, old_defer_size = 0;
322 : struct mem_cgroup *memcg;
323 :
324 : if (!root_mem_cgroup)
325 : goto out;
326 :
327 : lockdep_assert_held(&shrinker_mutex);
328 :
329 : map_size = shrinker_map_size(new_nr_max);
330 : defer_size = shrinker_defer_size(new_nr_max);
331 : old_map_size = shrinker_map_size(shrinker_nr_max);
332 : old_defer_size = shrinker_defer_size(shrinker_nr_max);
333 :
334 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
335 : do {
336 : ret = expand_one_shrinker_info(memcg, map_size, defer_size,
337 : old_map_size, old_defer_size,
338 : new_nr_max);
339 : if (ret) {
340 : mem_cgroup_iter_break(NULL, memcg);
341 : goto out;
342 : }
343 : } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
344 : out:
345 : if (!ret)
346 : shrinker_nr_max = new_nr_max;
347 :
348 : return ret;
349 : }
350 :
351 : void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
352 : {
353 : if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
354 : struct shrinker_info *info;
355 : int srcu_idx;
356 :
357 : srcu_idx = srcu_read_lock(&shrinker_srcu);
358 : info = shrinker_info_srcu(memcg, nid);
359 : if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {
360 : /* Pairs with smp mb in shrink_slab() */
361 : smp_mb__before_atomic();
362 : set_bit(shrinker_id, info->map);
363 : }
364 : srcu_read_unlock(&shrinker_srcu, srcu_idx);
365 : }
366 : }
367 :
368 : static DEFINE_IDR(shrinker_idr);
369 :
370 : static int prealloc_memcg_shrinker(struct shrinker *shrinker)
371 : {
372 : int id, ret = -ENOMEM;
373 :
374 : if (mem_cgroup_disabled())
375 : return -ENOSYS;
376 :
377 : mutex_lock(&shrinker_mutex);
378 : id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
379 : if (id < 0)
380 : goto unlock;
381 :
382 : if (id >= shrinker_nr_max) {
383 : if (expand_shrinker_info(id)) {
384 : idr_remove(&shrinker_idr, id);
385 : goto unlock;
386 : }
387 : }
388 : shrinker->id = id;
389 : ret = 0;
390 : unlock:
391 : mutex_unlock(&shrinker_mutex);
392 : return ret;
393 : }
394 :
395 : static void unregister_memcg_shrinker(struct shrinker *shrinker)
396 : {
397 : int id = shrinker->id;
398 :
399 : BUG_ON(id < 0);
400 :
401 : lockdep_assert_held(&shrinker_mutex);
402 :
403 : idr_remove(&shrinker_idr, id);
404 : }
405 :
406 : static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
407 : struct mem_cgroup *memcg)
408 : {
409 : struct shrinker_info *info;
410 :
411 : info = shrinker_info_srcu(memcg, nid);
412 : return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0);
413 : }
414 :
415 : static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
416 : struct mem_cgroup *memcg)
417 : {
418 : struct shrinker_info *info;
419 :
420 : info = shrinker_info_srcu(memcg, nid);
421 : return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]);
422 : }
423 :
424 : void reparent_shrinker_deferred(struct mem_cgroup *memcg)
425 : {
426 : int i, nid;
427 : long nr;
428 : struct mem_cgroup *parent;
429 : struct shrinker_info *child_info, *parent_info;
430 :
431 : parent = parent_mem_cgroup(memcg);
432 : if (!parent)
433 : parent = root_mem_cgroup;
434 :
435 : /* Prevent from concurrent shrinker_info expand */
436 : mutex_lock(&shrinker_mutex);
437 : for_each_node(nid) {
438 : child_info = shrinker_info_protected(memcg, nid);
439 : parent_info = shrinker_info_protected(parent, nid);
440 : for (i = 0; i < child_info->map_nr_max; i++) {
441 : nr = atomic_long_read(&child_info->nr_deferred[i]);
442 : atomic_long_add(nr, &parent_info->nr_deferred[i]);
443 : }
444 : }
445 : mutex_unlock(&shrinker_mutex);
446 : }
447 :
448 : static bool cgroup_reclaim(struct scan_control *sc)
449 : {
450 : return sc->target_mem_cgroup;
451 : }
452 :
453 : static bool global_reclaim(struct scan_control *sc)
454 : {
455 : return !sc->target_mem_cgroup || mem_cgroup_is_root(sc->target_mem_cgroup);
456 : }
457 :
458 : /**
459 : * writeback_throttling_sane - is the usual dirty throttling mechanism available?
460 : * @sc: scan_control in question
461 : *
462 : * The normal page dirty throttling mechanism in balance_dirty_pages() is
463 : * completely broken with the legacy memcg and direct stalling in
464 : * shrink_folio_list() is used for throttling instead, which lacks all the
465 : * niceties such as fairness, adaptive pausing, bandwidth proportional
466 : * allocation and configurability.
467 : *
468 : * This function tests whether the vmscan currently in progress can assume
469 : * that the normal dirty throttling mechanism is operational.
470 : */
471 : static bool writeback_throttling_sane(struct scan_control *sc)
472 : {
473 : if (!cgroup_reclaim(sc))
474 : return true;
475 : #ifdef CONFIG_CGROUP_WRITEBACK
476 : if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
477 : return true;
478 : #endif
479 : return false;
480 : }
481 : #else
482 : static int prealloc_memcg_shrinker(struct shrinker *shrinker)
483 : {
484 : return -ENOSYS;
485 : }
486 :
487 : static void unregister_memcg_shrinker(struct shrinker *shrinker)
488 : {
489 : }
490 :
491 : static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
492 : struct mem_cgroup *memcg)
493 : {
494 : return 0;
495 : }
496 :
497 : static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
498 : struct mem_cgroup *memcg)
499 : {
500 : return 0;
501 : }
502 :
503 : static bool cgroup_reclaim(struct scan_control *sc)
504 : {
505 : return false;
506 : }
507 :
508 : static bool global_reclaim(struct scan_control *sc)
509 : {
510 : return true;
511 : }
512 :
513 : static bool writeback_throttling_sane(struct scan_control *sc)
514 : {
515 : return true;
516 : }
517 : #endif
518 :
519 0 : static void set_task_reclaim_state(struct task_struct *task,
520 : struct reclaim_state *rs)
521 : {
522 : /* Check for an overwrite */
523 0 : WARN_ON_ONCE(rs && task->reclaim_state);
524 :
525 : /* Check for the nulling of an already-nulled member */
526 0 : WARN_ON_ONCE(!rs && !task->reclaim_state);
527 :
528 0 : task->reclaim_state = rs;
529 0 : }
530 :
531 : /*
532 : * flush_reclaim_state(): add pages reclaimed outside of LRU-based reclaim to
533 : * scan_control->nr_reclaimed.
534 : */
535 : static void flush_reclaim_state(struct scan_control *sc)
536 : {
537 : /*
538 : * Currently, reclaim_state->reclaimed includes three types of pages
539 : * freed outside of vmscan:
540 : * (1) Slab pages.
541 : * (2) Clean file pages from pruned inodes (on highmem systems).
542 : * (3) XFS freed buffer pages.
543 : *
544 : * For all of these cases, we cannot universally link the pages to a
545 : * single memcg. For example, a memcg-aware shrinker can free one object
546 : * charged to the target memcg, causing an entire page to be freed.
547 : * If we count the entire page as reclaimed from the memcg, we end up
548 : * overestimating the reclaimed amount (potentially under-reclaiming).
549 : *
550 : * Only count such pages for global reclaim to prevent under-reclaiming
551 : * from the target memcg; preventing unnecessary retries during memcg
552 : * charging and false positives from proactive reclaim.
553 : *
554 : * For uncommon cases where the freed pages were actually mostly
555 : * charged to the target memcg, we end up underestimating the reclaimed
556 : * amount. This should be fine. The freed pages will be uncharged
557 : * anyway, even if they are not counted here properly, and we will be
558 : * able to make forward progress in charging (which is usually in a
559 : * retry loop).
560 : *
561 : * We can go one step further, and report the uncharged objcg pages in
562 : * memcg reclaim, to make reporting more accurate and reduce
563 : * underestimation, but it's probably not worth the complexity for now.
564 : */
565 0 : if (current->reclaim_state && global_reclaim(sc)) {
566 0 : sc->nr_reclaimed += current->reclaim_state->reclaimed;
567 0 : current->reclaim_state->reclaimed = 0;
568 : }
569 : }
570 :
571 : static long xchg_nr_deferred(struct shrinker *shrinker,
572 : struct shrink_control *sc)
573 : {
574 0 : int nid = sc->nid;
575 :
576 0 : if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
577 0 : nid = 0;
578 :
579 0 : if (sc->memcg &&
580 0 : (shrinker->flags & SHRINKER_MEMCG_AWARE))
581 : return xchg_nr_deferred_memcg(nid, shrinker,
582 : sc->memcg);
583 :
584 0 : return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
585 : }
586 :
587 :
588 : static long add_nr_deferred(long nr, struct shrinker *shrinker,
589 : struct shrink_control *sc)
590 : {
591 0 : int nid = sc->nid;
592 :
593 0 : if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
594 0 : nid = 0;
595 :
596 0 : if (sc->memcg &&
597 0 : (shrinker->flags & SHRINKER_MEMCG_AWARE))
598 : return add_nr_deferred_memcg(nr, nid, shrinker,
599 : sc->memcg);
600 :
601 0 : return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);
602 : }
603 :
604 : static bool can_demote(int nid, struct scan_control *sc)
605 : {
606 : if (!numa_demotion_enabled)
607 : return false;
608 : if (sc && sc->no_demotion)
609 : return false;
610 : if (next_demotion_node(nid) == NUMA_NO_NODE)
611 : return false;
612 :
613 : return true;
614 : }
615 :
616 : static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
617 : int nid,
618 : struct scan_control *sc)
619 : {
620 : if (memcg == NULL) {
621 : /*
622 : * For non-memcg reclaim, is there
623 : * space in any swap device?
624 : */
625 0 : if (get_nr_swap_pages() > 0)
626 : return true;
627 : } else {
628 : /* Is the memcg below its swap limit? */
629 : if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
630 : return true;
631 : }
632 :
633 : /*
634 : * The page can not be swapped.
635 : *
636 : * Can it be reclaimed from this node via demotion?
637 : */
638 : return can_demote(nid, sc);
639 : }
640 :
641 : /*
642 : * This misses isolated folios which are not accounted for to save counters.
643 : * As the data only determines if reclaim or compaction continues, it is
644 : * not expected that isolated folios will be a dominating factor.
645 : */
646 0 : unsigned long zone_reclaimable_pages(struct zone *zone)
647 : {
648 : unsigned long nr;
649 :
650 0 : nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
651 0 : zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
652 0 : if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL))
653 0 : nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
654 0 : zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
655 :
656 0 : return nr;
657 : }
658 :
659 : /**
660 : * lruvec_lru_size - Returns the number of pages on the given LRU list.
661 : * @lruvec: lru vector
662 : * @lru: lru to use
663 : * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list)
664 : */
665 : static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
666 : int zone_idx)
667 : {
668 : unsigned long size = 0;
669 : int zid;
670 :
671 0 : for (zid = 0; zid <= zone_idx; zid++) {
672 0 : struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
673 :
674 0 : if (!managed_zone(zone))
675 0 : continue;
676 :
677 : if (!mem_cgroup_disabled())
678 : size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
679 : else
680 0 : size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
681 : }
682 : return size;
683 : }
684 :
685 : /*
686 : * Add a shrinker callback to be called from the vm.
687 : */
688 33 : static int __prealloc_shrinker(struct shrinker *shrinker)
689 : {
690 : unsigned int size;
691 : int err;
692 :
693 33 : if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
694 33 : err = prealloc_memcg_shrinker(shrinker);
695 : if (err != -ENOSYS)
696 : return err;
697 :
698 33 : shrinker->flags &= ~SHRINKER_MEMCG_AWARE;
699 : }
700 :
701 33 : size = sizeof(*shrinker->nr_deferred);
702 : if (shrinker->flags & SHRINKER_NUMA_AWARE)
703 : size *= nr_node_ids;
704 :
705 33 : shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
706 33 : if (!shrinker->nr_deferred)
707 : return -ENOMEM;
708 :
709 : return 0;
710 : }
711 :
712 : #ifdef CONFIG_SHRINKER_DEBUG
713 : int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...)
714 : {
715 : va_list ap;
716 : int err;
717 :
718 : va_start(ap, fmt);
719 : shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap);
720 : va_end(ap);
721 : if (!shrinker->name)
722 : return -ENOMEM;
723 :
724 : err = __prealloc_shrinker(shrinker);
725 : if (err) {
726 : kfree_const(shrinker->name);
727 : shrinker->name = NULL;
728 : }
729 :
730 : return err;
731 : }
732 : #else
733 33 : int prealloc_shrinker(struct shrinker *shrinker, const char *fmt, ...)
734 : {
735 33 : return __prealloc_shrinker(shrinker);
736 : }
737 : #endif
738 :
739 0 : void free_prealloced_shrinker(struct shrinker *shrinker)
740 : {
741 : #ifdef CONFIG_SHRINKER_DEBUG
742 : kfree_const(shrinker->name);
743 : shrinker->name = NULL;
744 : #endif
745 0 : if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
746 0 : mutex_lock(&shrinker_mutex);
747 0 : unregister_memcg_shrinker(shrinker);
748 0 : mutex_unlock(&shrinker_mutex);
749 0 : return;
750 : }
751 :
752 0 : kfree(shrinker->nr_deferred);
753 0 : shrinker->nr_deferred = NULL;
754 : }
755 :
756 33 : void register_shrinker_prepared(struct shrinker *shrinker)
757 : {
758 33 : mutex_lock(&shrinker_mutex);
759 66 : list_add_tail_rcu(&shrinker->list, &shrinker_list);
760 33 : shrinker->flags |= SHRINKER_REGISTERED;
761 33 : shrinker_debugfs_add(shrinker);
762 33 : mutex_unlock(&shrinker_mutex);
763 33 : }
764 :
765 0 : static int __register_shrinker(struct shrinker *shrinker)
766 : {
767 0 : int err = __prealloc_shrinker(shrinker);
768 :
769 0 : if (err)
770 : return err;
771 0 : register_shrinker_prepared(shrinker);
772 0 : return 0;
773 : }
774 :
775 : #ifdef CONFIG_SHRINKER_DEBUG
776 : int register_shrinker(struct shrinker *shrinker, const char *fmt, ...)
777 : {
778 : va_list ap;
779 : int err;
780 :
781 : va_start(ap, fmt);
782 : shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap);
783 : va_end(ap);
784 : if (!shrinker->name)
785 : return -ENOMEM;
786 :
787 : err = __register_shrinker(shrinker);
788 : if (err) {
789 : kfree_const(shrinker->name);
790 : shrinker->name = NULL;
791 : }
792 : return err;
793 : }
794 : #else
795 0 : int register_shrinker(struct shrinker *shrinker, const char *fmt, ...)
796 : {
797 0 : return __register_shrinker(shrinker);
798 : }
799 : #endif
800 : EXPORT_SYMBOL(register_shrinker);
801 :
802 : /*
803 : * Remove one
804 : */
805 22 : void unregister_shrinker(struct shrinker *shrinker)
806 : {
807 : struct dentry *debugfs_entry;
808 : int debugfs_id;
809 :
810 22 : if (!(shrinker->flags & SHRINKER_REGISTERED))
811 : return;
812 :
813 22 : mutex_lock(&shrinker_mutex);
814 44 : list_del_rcu(&shrinker->list);
815 22 : shrinker->flags &= ~SHRINKER_REGISTERED;
816 : if (shrinker->flags & SHRINKER_MEMCG_AWARE)
817 : unregister_memcg_shrinker(shrinker);
818 44 : debugfs_entry = shrinker_debugfs_detach(shrinker, &debugfs_id);
819 22 : mutex_unlock(&shrinker_mutex);
820 :
821 22 : atomic_inc(&shrinker_srcu_generation);
822 22 : synchronize_srcu(&shrinker_srcu);
823 :
824 22 : shrinker_debugfs_remove(debugfs_entry, debugfs_id);
825 :
826 22 : kfree(shrinker->nr_deferred);
827 22 : shrinker->nr_deferred = NULL;
828 : }
829 : EXPORT_SYMBOL(unregister_shrinker);
830 :
831 : /**
832 : * synchronize_shrinkers - Wait for all running shrinkers to complete.
833 : *
834 : * This is useful to guarantee that all shrinker invocations have seen an
835 : * update, before freeing memory.
836 : */
837 0 : void synchronize_shrinkers(void)
838 : {
839 0 : atomic_inc(&shrinker_srcu_generation);
840 0 : synchronize_srcu(&shrinker_srcu);
841 0 : }
842 : EXPORT_SYMBOL(synchronize_shrinkers);
843 :
844 : #define SHRINK_BATCH 128
845 :
846 0 : static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
847 : struct shrinker *shrinker, int priority)
848 : {
849 0 : unsigned long freed = 0;
850 : unsigned long long delta;
851 : long total_scan;
852 : long freeable;
853 : long nr;
854 : long new_nr;
855 0 : long batch_size = shrinker->batch ? shrinker->batch
856 0 : : SHRINK_BATCH;
857 0 : long scanned = 0, next_deferred;
858 :
859 0 : freeable = shrinker->count_objects(shrinker, shrinkctl);
860 0 : if (freeable == 0 || freeable == SHRINK_EMPTY)
861 : return freeable;
862 :
863 : /*
864 : * copy the current shrinker scan count into a local variable
865 : * and zero it so that other concurrent shrinker invocations
866 : * don't also do this scanning work.
867 : */
868 0 : nr = xchg_nr_deferred(shrinker, shrinkctl);
869 :
870 0 : if (shrinker->seeks) {
871 0 : delta = freeable >> priority;
872 0 : delta *= 4;
873 0 : do_div(delta, shrinker->seeks);
874 : } else {
875 : /*
876 : * These objects don't require any IO to create. Trim
877 : * them aggressively under memory pressure to keep
878 : * them from causing refetches in the IO caches.
879 : */
880 0 : delta = freeable / 2;
881 : }
882 :
883 0 : total_scan = nr >> priority;
884 0 : total_scan += delta;
885 0 : total_scan = min(total_scan, (2 * freeable));
886 :
887 0 : trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
888 : freeable, delta, total_scan, priority);
889 :
890 : /*
891 : * Normally, we should not scan less than batch_size objects in one
892 : * pass to avoid too frequent shrinker calls, but if the slab has less
893 : * than batch_size objects in total and we are really tight on memory,
894 : * we will try to reclaim all available objects, otherwise we can end
895 : * up failing allocations although there are plenty of reclaimable
896 : * objects spread over several slabs with usage less than the
897 : * batch_size.
898 : *
899 : * We detect the "tight on memory" situations by looking at the total
900 : * number of objects we want to scan (total_scan). If it is greater
901 : * than the total number of objects on slab (freeable), we must be
902 : * scanning at high prio and therefore should try to reclaim as much as
903 : * possible.
904 : */
905 0 : while (total_scan >= batch_size ||
906 0 : total_scan >= freeable) {
907 : unsigned long ret;
908 0 : unsigned long nr_to_scan = min(batch_size, total_scan);
909 :
910 0 : shrinkctl->nr_to_scan = nr_to_scan;
911 0 : shrinkctl->nr_scanned = nr_to_scan;
912 0 : ret = shrinker->scan_objects(shrinker, shrinkctl);
913 0 : if (ret == SHRINK_STOP)
914 : break;
915 0 : freed += ret;
916 :
917 0 : count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
918 0 : total_scan -= shrinkctl->nr_scanned;
919 0 : scanned += shrinkctl->nr_scanned;
920 :
921 0 : cond_resched();
922 : }
923 :
924 : /*
925 : * The deferred work is increased by any new work (delta) that wasn't
926 : * done, decreased by old deferred work that was done now.
927 : *
928 : * And it is capped to two times of the freeable items.
929 : */
930 0 : next_deferred = max_t(long, (nr + delta - scanned), 0);
931 0 : next_deferred = min(next_deferred, (2 * freeable));
932 :
933 : /*
934 : * move the unused scan count back into the shrinker in a
935 : * manner that handles concurrent updates.
936 : */
937 0 : new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl);
938 :
939 0 : trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan);
940 0 : return freed;
941 : }
942 :
943 : #ifdef CONFIG_MEMCG
944 : static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
945 : struct mem_cgroup *memcg, int priority)
946 : {
947 : struct shrinker_info *info;
948 : unsigned long ret, freed = 0;
949 : int srcu_idx, generation;
950 : int i = 0;
951 :
952 : if (!mem_cgroup_online(memcg))
953 : return 0;
954 :
955 : again:
956 : srcu_idx = srcu_read_lock(&shrinker_srcu);
957 : info = shrinker_info_srcu(memcg, nid);
958 : if (unlikely(!info))
959 : goto unlock;
960 :
961 : generation = atomic_read(&shrinker_srcu_generation);
962 : for_each_set_bit_from(i, info->map, info->map_nr_max) {
963 : struct shrink_control sc = {
964 : .gfp_mask = gfp_mask,
965 : .nid = nid,
966 : .memcg = memcg,
967 : };
968 : struct shrinker *shrinker;
969 :
970 : shrinker = idr_find(&shrinker_idr, i);
971 : if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) {
972 : if (!shrinker)
973 : clear_bit(i, info->map);
974 : continue;
975 : }
976 :
977 : /* Call non-slab shrinkers even though kmem is disabled */
978 : if (!memcg_kmem_online() &&
979 : !(shrinker->flags & SHRINKER_NONSLAB))
980 : continue;
981 :
982 : ret = do_shrink_slab(&sc, shrinker, priority);
983 : if (ret == SHRINK_EMPTY) {
984 : clear_bit(i, info->map);
985 : /*
986 : * After the shrinker reported that it had no objects to
987 : * free, but before we cleared the corresponding bit in
988 : * the memcg shrinker map, a new object might have been
989 : * added. To make sure, we have the bit set in this
990 : * case, we invoke the shrinker one more time and reset
991 : * the bit if it reports that it is not empty anymore.
992 : * The memory barrier here pairs with the barrier in
993 : * set_shrinker_bit():
994 : *
995 : * list_lru_add() shrink_slab_memcg()
996 : * list_add_tail() clear_bit()
997 : * <MB> <MB>
998 : * set_bit() do_shrink_slab()
999 : */
1000 : smp_mb__after_atomic();
1001 : ret = do_shrink_slab(&sc, shrinker, priority);
1002 : if (ret == SHRINK_EMPTY)
1003 : ret = 0;
1004 : else
1005 : set_shrinker_bit(memcg, nid, i);
1006 : }
1007 : freed += ret;
1008 : if (atomic_read(&shrinker_srcu_generation) != generation) {
1009 : srcu_read_unlock(&shrinker_srcu, srcu_idx);
1010 : i++;
1011 : goto again;
1012 : }
1013 : }
1014 : unlock:
1015 : srcu_read_unlock(&shrinker_srcu, srcu_idx);
1016 : return freed;
1017 : }
1018 : #else /* CONFIG_MEMCG */
1019 : static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
1020 : struct mem_cgroup *memcg, int priority)
1021 : {
1022 : return 0;
1023 : }
1024 : #endif /* CONFIG_MEMCG */
1025 :
1026 : /**
1027 : * shrink_slab - shrink slab caches
1028 : * @gfp_mask: allocation context
1029 : * @nid: node whose slab caches to target
1030 : * @memcg: memory cgroup whose slab caches to target
1031 : * @priority: the reclaim priority
1032 : *
1033 : * Call the shrink functions to age shrinkable caches.
1034 : *
1035 : * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
1036 : * unaware shrinkers will receive a node id of 0 instead.
1037 : *
1038 : * @memcg specifies the memory cgroup to target. Unaware shrinkers
1039 : * are called only if it is the root cgroup.
1040 : *
1041 : * @priority is sc->priority, we take the number of objects and >> by priority
1042 : * in order to get the scan target.
1043 : *
1044 : * Returns the number of reclaimed slab objects.
1045 : */
1046 0 : static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
1047 : struct mem_cgroup *memcg,
1048 : int priority)
1049 : {
1050 0 : unsigned long ret, freed = 0;
1051 : struct shrinker *shrinker;
1052 : int srcu_idx, generation;
1053 :
1054 : /*
1055 : * The root memcg might be allocated even though memcg is disabled
1056 : * via "cgroup_disable=memory" boot parameter. This could make
1057 : * mem_cgroup_is_root() return false, then just run memcg slab
1058 : * shrink, but skip global shrink. This may result in premature
1059 : * oom.
1060 : */
1061 : if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
1062 : return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
1063 :
1064 0 : srcu_idx = srcu_read_lock(&shrinker_srcu);
1065 :
1066 0 : generation = atomic_read(&shrinker_srcu_generation);
1067 0 : list_for_each_entry_srcu(shrinker, &shrinker_list, list,
1068 : srcu_read_lock_held(&shrinker_srcu)) {
1069 0 : struct shrink_control sc = {
1070 : .gfp_mask = gfp_mask,
1071 : .nid = nid,
1072 : .memcg = memcg,
1073 : };
1074 :
1075 0 : ret = do_shrink_slab(&sc, shrinker, priority);
1076 0 : if (ret == SHRINK_EMPTY)
1077 0 : ret = 0;
1078 0 : freed += ret;
1079 :
1080 0 : if (atomic_read(&shrinker_srcu_generation) != generation) {
1081 0 : freed = freed ? : 1;
1082 0 : break;
1083 : }
1084 : }
1085 :
1086 0 : srcu_read_unlock(&shrinker_srcu, srcu_idx);
1087 0 : cond_resched();
1088 : return freed;
1089 : }
1090 :
1091 : static unsigned long drop_slab_node(int nid)
1092 : {
1093 0 : unsigned long freed = 0;
1094 0 : struct mem_cgroup *memcg = NULL;
1095 :
1096 0 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
1097 : do {
1098 0 : freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
1099 0 : } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
1100 :
1101 : return freed;
1102 : }
1103 :
1104 0 : void drop_slab(void)
1105 : {
1106 : int nid;
1107 0 : int shift = 0;
1108 : unsigned long freed;
1109 :
1110 : do {
1111 0 : freed = 0;
1112 0 : for_each_online_node(nid) {
1113 0 : if (fatal_signal_pending(current))
1114 : return;
1115 :
1116 0 : freed += drop_slab_node(nid);
1117 : }
1118 0 : } while ((freed >> shift++) > 1);
1119 : }
1120 :
1121 : static int reclaimer_offset(void)
1122 : {
1123 : BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
1124 : PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD);
1125 : BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
1126 : PGSCAN_DIRECT - PGSCAN_KSWAPD);
1127 : BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
1128 : PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD);
1129 : BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
1130 : PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD);
1131 :
1132 0 : if (current_is_kswapd())
1133 : return 0;
1134 : if (current_is_khugepaged())
1135 : return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD;
1136 : return PGSTEAL_DIRECT - PGSTEAL_KSWAPD;
1137 : }
1138 :
1139 : static inline int is_page_cache_freeable(struct folio *folio)
1140 : {
1141 : /*
1142 : * A freeable page cache folio is referenced only by the caller
1143 : * that isolated the folio, the page cache and optional filesystem
1144 : * private data at folio->private.
1145 : */
1146 0 : return folio_ref_count(folio) - folio_test_private(folio) ==
1147 0 : 1 + folio_nr_pages(folio);
1148 : }
1149 :
1150 : /*
1151 : * We detected a synchronous write error writing a folio out. Probably
1152 : * -ENOSPC. We need to propagate that into the address_space for a subsequent
1153 : * fsync(), msync() or close().
1154 : *
1155 : * The tricky part is that after writepage we cannot touch the mapping: nothing
1156 : * prevents it from being freed up. But we have a ref on the folio and once
1157 : * that folio is locked, the mapping is pinned.
1158 : *
1159 : * We're allowed to run sleeping folio_lock() here because we know the caller has
1160 : * __GFP_FS.
1161 : */
1162 0 : static void handle_write_error(struct address_space *mapping,
1163 : struct folio *folio, int error)
1164 : {
1165 0 : folio_lock(folio);
1166 0 : if (folio_mapping(folio) == mapping)
1167 0 : mapping_set_error(mapping, error);
1168 0 : folio_unlock(folio);
1169 0 : }
1170 :
1171 0 : static bool skip_throttle_noprogress(pg_data_t *pgdat)
1172 : {
1173 0 : int reclaimable = 0, write_pending = 0;
1174 : int i;
1175 :
1176 : /*
1177 : * If kswapd is disabled, reschedule if necessary but do not
1178 : * throttle as the system is likely near OOM.
1179 : */
1180 0 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
1181 : return true;
1182 :
1183 : /*
1184 : * If there are a lot of dirty/writeback folios then do not
1185 : * throttle as throttling will occur when the folios cycle
1186 : * towards the end of the LRU if still under writeback.
1187 : */
1188 0 : for (i = 0; i < MAX_NR_ZONES; i++) {
1189 0 : struct zone *zone = pgdat->node_zones + i;
1190 :
1191 0 : if (!managed_zone(zone))
1192 0 : continue;
1193 :
1194 0 : reclaimable += zone_reclaimable_pages(zone);
1195 0 : write_pending += zone_page_state_snapshot(zone,
1196 : NR_ZONE_WRITE_PENDING);
1197 : }
1198 0 : if (2 * write_pending <= reclaimable)
1199 : return true;
1200 :
1201 0 : return false;
1202 : }
1203 :
1204 0 : void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)
1205 : {
1206 0 : wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason];
1207 : long timeout, ret;
1208 0 : DEFINE_WAIT(wait);
1209 :
1210 : /*
1211 : * Do not throttle user workers, kthreads other than kswapd or
1212 : * workqueues. They may be required for reclaim to make
1213 : * forward progress (e.g. journalling workqueues or kthreads).
1214 : */
1215 0 : if (!current_is_kswapd() &&
1216 0 : current->flags & (PF_USER_WORKER|PF_KTHREAD)) {
1217 0 : cond_resched();
1218 0 : return;
1219 : }
1220 :
1221 : /*
1222 : * These figures are pulled out of thin air.
1223 : * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many
1224 : * parallel reclaimers which is a short-lived event so the timeout is
1225 : * short. Failing to make progress or waiting on writeback are
1226 : * potentially long-lived events so use a longer timeout. This is shaky
1227 : * logic as a failure to make progress could be due to anything from
1228 : * writeback to a slow device to excessive referenced folios at the tail
1229 : * of the inactive LRU.
1230 : */
1231 0 : switch(reason) {
1232 : case VMSCAN_THROTTLE_WRITEBACK:
1233 0 : timeout = HZ/10;
1234 :
1235 0 : if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) {
1236 0 : WRITE_ONCE(pgdat->nr_reclaim_start,
1237 : node_page_state(pgdat, NR_THROTTLED_WRITTEN));
1238 : }
1239 :
1240 : break;
1241 : case VMSCAN_THROTTLE_CONGESTED:
1242 : fallthrough;
1243 : case VMSCAN_THROTTLE_NOPROGRESS:
1244 0 : if (skip_throttle_noprogress(pgdat)) {
1245 0 : cond_resched();
1246 0 : return;
1247 : }
1248 :
1249 : timeout = 1;
1250 :
1251 : break;
1252 : case VMSCAN_THROTTLE_ISOLATED:
1253 : timeout = HZ/50;
1254 : break;
1255 : default:
1256 0 : WARN_ON_ONCE(1);
1257 : timeout = HZ;
1258 : break;
1259 : }
1260 :
1261 0 : prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1262 0 : ret = schedule_timeout(timeout);
1263 0 : finish_wait(wqh, &wait);
1264 :
1265 0 : if (reason == VMSCAN_THROTTLE_WRITEBACK)
1266 0 : atomic_dec(&pgdat->nr_writeback_throttled);
1267 :
1268 0 : trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout),
1269 0 : jiffies_to_usecs(timeout - ret),
1270 : reason);
1271 : }
1272 :
1273 : /*
1274 : * Account for folios written if tasks are throttled waiting on dirty
1275 : * folios to clean. If enough folios have been cleaned since throttling
1276 : * started then wakeup the throttled tasks.
1277 : */
1278 0 : void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
1279 : int nr_throttled)
1280 : {
1281 : unsigned long nr_written;
1282 :
1283 0 : node_stat_add_folio(folio, NR_THROTTLED_WRITTEN);
1284 :
1285 : /*
1286 : * This is an inaccurate read as the per-cpu deltas may not
1287 : * be synchronised. However, given that the system is
1288 : * writeback throttled, it is not worth taking the penalty
1289 : * of getting an accurate count. At worst, the throttle
1290 : * timeout guarantees forward progress.
1291 : */
1292 0 : nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) -
1293 0 : READ_ONCE(pgdat->nr_reclaim_start);
1294 :
1295 0 : if (nr_written > SWAP_CLUSTER_MAX * nr_throttled)
1296 0 : wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]);
1297 0 : }
1298 :
1299 : /* possible outcome of pageout() */
1300 : typedef enum {
1301 : /* failed to write folio out, folio is locked */
1302 : PAGE_KEEP,
1303 : /* move folio to the active list, folio is locked */
1304 : PAGE_ACTIVATE,
1305 : /* folio has been sent to the disk successfully, folio is unlocked */
1306 : PAGE_SUCCESS,
1307 : /* folio is clean and locked */
1308 : PAGE_CLEAN,
1309 : } pageout_t;
1310 :
1311 : /*
1312 : * pageout is called by shrink_folio_list() for each dirty folio.
1313 : * Calls ->writepage().
1314 : */
1315 0 : static pageout_t pageout(struct folio *folio, struct address_space *mapping,
1316 : struct swap_iocb **plug)
1317 : {
1318 : /*
1319 : * If the folio is dirty, only perform writeback if that write
1320 : * will be non-blocking. To prevent this allocation from being
1321 : * stalled by pagecache activity. But note that there may be
1322 : * stalls if we need to run get_block(). We could test
1323 : * PagePrivate for that.
1324 : *
1325 : * If this process is currently in __generic_file_write_iter() against
1326 : * this folio's queue, we can perform writeback even if that
1327 : * will block.
1328 : *
1329 : * If the folio is swapcache, write it back even if that would
1330 : * block, for some throttling. This happens by accident, because
1331 : * swap_backing_dev_info is bust: it doesn't reflect the
1332 : * congestion state of the swapdevs. Easy to fix, if needed.
1333 : */
1334 0 : if (!is_page_cache_freeable(folio))
1335 : return PAGE_KEEP;
1336 0 : if (!mapping) {
1337 : /*
1338 : * Some data journaling orphaned folios can have
1339 : * folio->mapping == NULL while being dirty with clean buffers.
1340 : */
1341 0 : if (folio_test_private(folio)) {
1342 0 : if (try_to_free_buffers(folio)) {
1343 0 : folio_clear_dirty(folio);
1344 0 : pr_info("%s: orphaned folio\n", __func__);
1345 0 : return PAGE_CLEAN;
1346 : }
1347 : }
1348 : return PAGE_KEEP;
1349 : }
1350 0 : if (mapping->a_ops->writepage == NULL)
1351 : return PAGE_ACTIVATE;
1352 :
1353 0 : if (folio_clear_dirty_for_io(folio)) {
1354 : int res;
1355 0 : struct writeback_control wbc = {
1356 : .sync_mode = WB_SYNC_NONE,
1357 : .nr_to_write = SWAP_CLUSTER_MAX,
1358 : .range_start = 0,
1359 : .range_end = LLONG_MAX,
1360 : .for_reclaim = 1,
1361 : .swap_plug = plug,
1362 : };
1363 :
1364 0 : folio_set_reclaim(folio);
1365 0 : res = mapping->a_ops->writepage(&folio->page, &wbc);
1366 0 : if (res < 0)
1367 0 : handle_write_error(mapping, folio, res);
1368 0 : if (res == AOP_WRITEPAGE_ACTIVATE) {
1369 0 : folio_clear_reclaim(folio);
1370 0 : return PAGE_ACTIVATE;
1371 : }
1372 :
1373 0 : if (!folio_test_writeback(folio)) {
1374 : /* synchronous write or broken a_ops? */
1375 : folio_clear_reclaim(folio);
1376 : }
1377 0 : trace_mm_vmscan_write_folio(folio);
1378 0 : node_stat_add_folio(folio, NR_VMSCAN_WRITE);
1379 0 : return PAGE_SUCCESS;
1380 : }
1381 :
1382 : return PAGE_CLEAN;
1383 : }
1384 :
1385 : /*
1386 : * Same as remove_mapping, but if the folio is removed from the mapping, it
1387 : * gets returned with a refcount of 0.
1388 : */
1389 0 : static int __remove_mapping(struct address_space *mapping, struct folio *folio,
1390 : bool reclaimed, struct mem_cgroup *target_memcg)
1391 : {
1392 : int refcount;
1393 0 : void *shadow = NULL;
1394 :
1395 0 : BUG_ON(!folio_test_locked(folio));
1396 0 : BUG_ON(mapping != folio_mapping(folio));
1397 :
1398 0 : if (!folio_test_swapcache(folio))
1399 0 : spin_lock(&mapping->host->i_lock);
1400 0 : xa_lock_irq(&mapping->i_pages);
1401 : /*
1402 : * The non racy check for a busy folio.
1403 : *
1404 : * Must be careful with the order of the tests. When someone has
1405 : * a ref to the folio, it may be possible that they dirty it then
1406 : * drop the reference. So if the dirty flag is tested before the
1407 : * refcount here, then the following race may occur:
1408 : *
1409 : * get_user_pages(&page);
1410 : * [user mapping goes away]
1411 : * write_to(page);
1412 : * !folio_test_dirty(folio) [good]
1413 : * folio_set_dirty(folio);
1414 : * folio_put(folio);
1415 : * !refcount(folio) [good, discard it]
1416 : *
1417 : * [oops, our write_to data is lost]
1418 : *
1419 : * Reversing the order of the tests ensures such a situation cannot
1420 : * escape unnoticed. The smp_rmb is needed to ensure the folio->flags
1421 : * load is not satisfied before that of folio->_refcount.
1422 : *
1423 : * Note that if the dirty flag is always set via folio_mark_dirty,
1424 : * and thus under the i_pages lock, then this ordering is not required.
1425 : */
1426 0 : refcount = 1 + folio_nr_pages(folio);
1427 0 : if (!folio_ref_freeze(folio, refcount))
1428 : goto cannot_free;
1429 : /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */
1430 0 : if (unlikely(folio_test_dirty(folio))) {
1431 : folio_ref_unfreeze(folio, refcount);
1432 : goto cannot_free;
1433 : }
1434 :
1435 0 : if (folio_test_swapcache(folio)) {
1436 0 : swp_entry_t swap = folio_swap_entry(folio);
1437 :
1438 0 : if (reclaimed && !mapping_exiting(mapping))
1439 0 : shadow = workingset_eviction(folio, target_memcg);
1440 0 : __delete_from_swap_cache(folio, swap, shadow);
1441 0 : mem_cgroup_swapout(folio, swap);
1442 0 : xa_unlock_irq(&mapping->i_pages);
1443 0 : put_swap_folio(folio, swap);
1444 : } else {
1445 : void (*free_folio)(struct folio *);
1446 :
1447 0 : free_folio = mapping->a_ops->free_folio;
1448 : /*
1449 : * Remember a shadow entry for reclaimed file cache in
1450 : * order to detect refaults, thus thrashing, later on.
1451 : *
1452 : * But don't store shadows in an address space that is
1453 : * already exiting. This is not just an optimization,
1454 : * inode reclaim needs to empty out the radix tree or
1455 : * the nodes are lost. Don't plant shadows behind its
1456 : * back.
1457 : *
1458 : * We also don't store shadows for DAX mappings because the
1459 : * only page cache folios found in these are zero pages
1460 : * covering holes, and because we don't want to mix DAX
1461 : * exceptional entries and shadow exceptional entries in the
1462 : * same address_space.
1463 : */
1464 0 : if (reclaimed && folio_is_file_lru(folio) &&
1465 0 : !mapping_exiting(mapping) && !dax_mapping(mapping))
1466 0 : shadow = workingset_eviction(folio, target_memcg);
1467 0 : __filemap_remove_folio(folio, shadow);
1468 0 : xa_unlock_irq(&mapping->i_pages);
1469 0 : if (mapping_shrinkable(mapping))
1470 0 : inode_add_lru(mapping->host);
1471 0 : spin_unlock(&mapping->host->i_lock);
1472 :
1473 0 : if (free_folio)
1474 0 : free_folio(folio);
1475 : }
1476 :
1477 : return 1;
1478 :
1479 : cannot_free:
1480 0 : xa_unlock_irq(&mapping->i_pages);
1481 0 : if (!folio_test_swapcache(folio))
1482 0 : spin_unlock(&mapping->host->i_lock);
1483 : return 0;
1484 : }
1485 :
1486 : /**
1487 : * remove_mapping() - Attempt to remove a folio from its mapping.
1488 : * @mapping: The address space.
1489 : * @folio: The folio to remove.
1490 : *
1491 : * If the folio is dirty, under writeback or if someone else has a ref
1492 : * on it, removal will fail.
1493 : * Return: The number of pages removed from the mapping. 0 if the folio
1494 : * could not be removed.
1495 : * Context: The caller should have a single refcount on the folio and
1496 : * hold its lock.
1497 : */
1498 0 : long remove_mapping(struct address_space *mapping, struct folio *folio)
1499 : {
1500 0 : if (__remove_mapping(mapping, folio, false, NULL)) {
1501 : /*
1502 : * Unfreezing the refcount with 1 effectively
1503 : * drops the pagecache ref for us without requiring another
1504 : * atomic operation.
1505 : */
1506 0 : folio_ref_unfreeze(folio, 1);
1507 : return folio_nr_pages(folio);
1508 : }
1509 : return 0;
1510 : }
1511 :
1512 : /**
1513 : * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
1514 : * @folio: Folio to be returned to an LRU list.
1515 : *
1516 : * Add previously isolated @folio to appropriate LRU list.
1517 : * The folio may still be unevictable for other reasons.
1518 : *
1519 : * Context: lru_lock must not be held, interrupts must be enabled.
1520 : */
1521 0 : void folio_putback_lru(struct folio *folio)
1522 : {
1523 0 : folio_add_lru(folio);
1524 0 : folio_put(folio); /* drop ref from isolate */
1525 0 : }
1526 :
1527 : enum folio_references {
1528 : FOLIOREF_RECLAIM,
1529 : FOLIOREF_RECLAIM_CLEAN,
1530 : FOLIOREF_KEEP,
1531 : FOLIOREF_ACTIVATE,
1532 : };
1533 :
1534 0 : static enum folio_references folio_check_references(struct folio *folio,
1535 : struct scan_control *sc)
1536 : {
1537 : int referenced_ptes, referenced_folio;
1538 : unsigned long vm_flags;
1539 :
1540 0 : referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
1541 : &vm_flags);
1542 0 : referenced_folio = folio_test_clear_referenced(folio);
1543 :
1544 : /*
1545 : * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
1546 : * Let the folio, now marked Mlocked, be moved to the unevictable list.
1547 : */
1548 0 : if (vm_flags & VM_LOCKED)
1549 : return FOLIOREF_ACTIVATE;
1550 :
1551 : /* rmap lock contention: rotate */
1552 0 : if (referenced_ptes == -1)
1553 : return FOLIOREF_KEEP;
1554 :
1555 0 : if (referenced_ptes) {
1556 : /*
1557 : * All mapped folios start out with page table
1558 : * references from the instantiating fault, so we need
1559 : * to look twice if a mapped file/anon folio is used more
1560 : * than once.
1561 : *
1562 : * Mark it and spare it for another trip around the
1563 : * inactive list. Another page table reference will
1564 : * lead to its activation.
1565 : *
1566 : * Note: the mark is set for activated folios as well
1567 : * so that recently deactivated but used folios are
1568 : * quickly recovered.
1569 : */
1570 0 : folio_set_referenced(folio);
1571 :
1572 0 : if (referenced_folio || referenced_ptes > 1)
1573 : return FOLIOREF_ACTIVATE;
1574 :
1575 : /*
1576 : * Activate file-backed executable folios after first usage.
1577 : */
1578 0 : if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio))
1579 : return FOLIOREF_ACTIVATE;
1580 :
1581 : return FOLIOREF_KEEP;
1582 : }
1583 :
1584 : /* Reclaim if clean, defer dirty folios to writeback */
1585 0 : if (referenced_folio && folio_is_file_lru(folio))
1586 : return FOLIOREF_RECLAIM_CLEAN;
1587 :
1588 : return FOLIOREF_RECLAIM;
1589 : }
1590 :
1591 : /* Check if a folio is dirty or under writeback */
1592 0 : static void folio_check_dirty_writeback(struct folio *folio,
1593 : bool *dirty, bool *writeback)
1594 : {
1595 : struct address_space *mapping;
1596 :
1597 : /*
1598 : * Anonymous folios are not handled by flushers and must be written
1599 : * from reclaim context. Do not stall reclaim based on them.
1600 : * MADV_FREE anonymous folios are put into inactive file list too.
1601 : * They could be mistakenly treated as file lru. So further anon
1602 : * test is needed.
1603 : */
1604 0 : if (!folio_is_file_lru(folio) ||
1605 0 : (folio_test_anon(folio) && !folio_test_swapbacked(folio))) {
1606 0 : *dirty = false;
1607 0 : *writeback = false;
1608 0 : return;
1609 : }
1610 :
1611 : /* By default assume that the folio flags are accurate */
1612 0 : *dirty = folio_test_dirty(folio);
1613 0 : *writeback = folio_test_writeback(folio);
1614 :
1615 : /* Verify dirty/writeback state if the filesystem supports it */
1616 0 : if (!folio_test_private(folio))
1617 : return;
1618 :
1619 0 : mapping = folio_mapping(folio);
1620 0 : if (mapping && mapping->a_ops->is_dirty_writeback)
1621 0 : mapping->a_ops->is_dirty_writeback(folio, dirty, writeback);
1622 : }
1623 :
1624 : static struct page *alloc_demote_page(struct page *page, unsigned long private)
1625 : {
1626 : struct page *target_page;
1627 : nodemask_t *allowed_mask;
1628 : struct migration_target_control *mtc;
1629 :
1630 : mtc = (struct migration_target_control *)private;
1631 :
1632 : allowed_mask = mtc->nmask;
1633 : /*
1634 : * make sure we allocate from the target node first also trying to
1635 : * demote or reclaim pages from the target node via kswapd if we are
1636 : * low on free memory on target node. If we don't do this and if
1637 : * we have free memory on the slower(lower) memtier, we would start
1638 : * allocating pages from slower(lower) memory tiers without even forcing
1639 : * a demotion of cold pages from the target memtier. This can result
1640 : * in the kernel placing hot pages in slower(lower) memory tiers.
1641 : */
1642 : mtc->nmask = NULL;
1643 : mtc->gfp_mask |= __GFP_THISNODE;
1644 : target_page = alloc_migration_target(page, (unsigned long)mtc);
1645 : if (target_page)
1646 : return target_page;
1647 :
1648 : mtc->gfp_mask &= ~__GFP_THISNODE;
1649 : mtc->nmask = allowed_mask;
1650 :
1651 : return alloc_migration_target(page, (unsigned long)mtc);
1652 : }
1653 :
1654 : /*
1655 : * Take folios on @demote_folios and attempt to demote them to another node.
1656 : * Folios which are not demoted are left on @demote_folios.
1657 : */
1658 : static unsigned int demote_folio_list(struct list_head *demote_folios,
1659 : struct pglist_data *pgdat)
1660 : {
1661 0 : int target_nid = next_demotion_node(pgdat->node_id);
1662 : unsigned int nr_succeeded;
1663 : nodemask_t allowed_mask;
1664 :
1665 0 : struct migration_target_control mtc = {
1666 : /*
1667 : * Allocate from 'node', or fail quickly and quietly.
1668 : * When this happens, 'page' will likely just be discarded
1669 : * instead of migrated.
1670 : */
1671 : .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN |
1672 : __GFP_NOMEMALLOC | GFP_NOWAIT,
1673 : .nid = target_nid,
1674 : .nmask = &allowed_mask
1675 : };
1676 :
1677 0 : if (list_empty(demote_folios))
1678 : return 0;
1679 :
1680 : if (target_nid == NUMA_NO_NODE)
1681 : return 0;
1682 :
1683 : node_get_allowed_targets(pgdat, &allowed_mask);
1684 :
1685 : /* Demotion ignores all cpuset and mempolicy settings */
1686 : migrate_pages(demote_folios, alloc_demote_page, NULL,
1687 : (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
1688 : &nr_succeeded);
1689 :
1690 : __count_vm_events(PGDEMOTE_KSWAPD + reclaimer_offset(), nr_succeeded);
1691 :
1692 : return nr_succeeded;
1693 : }
1694 :
1695 0 : static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)
1696 : {
1697 0 : if (gfp_mask & __GFP_FS)
1698 : return true;
1699 0 : if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO))
1700 : return false;
1701 : /*
1702 : * We can "enter_fs" for swap-cache with only __GFP_IO
1703 : * providing this isn't SWP_FS_OPS.
1704 : * ->flags can be updated non-atomicially (scan_swap_map_slots),
1705 : * but that will never affect SWP_FS_OPS, so the data_race
1706 : * is safe.
1707 : */
1708 0 : return !data_race(folio_swap_flags(folio) & SWP_FS_OPS);
1709 : }
1710 :
1711 : /*
1712 : * shrink_folio_list() returns the number of reclaimed pages
1713 : */
1714 0 : static unsigned int shrink_folio_list(struct list_head *folio_list,
1715 : struct pglist_data *pgdat, struct scan_control *sc,
1716 : struct reclaim_stat *stat, bool ignore_references)
1717 : {
1718 0 : LIST_HEAD(ret_folios);
1719 0 : LIST_HEAD(free_folios);
1720 0 : LIST_HEAD(demote_folios);
1721 0 : unsigned int nr_reclaimed = 0;
1722 0 : unsigned int pgactivate = 0;
1723 : bool do_demote_pass;
1724 0 : struct swap_iocb *plug = NULL;
1725 :
1726 0 : memset(stat, 0, sizeof(*stat));
1727 0 : cond_resched();
1728 0 : do_demote_pass = can_demote(pgdat->node_id, sc);
1729 :
1730 : retry:
1731 0 : while (!list_empty(folio_list)) {
1732 : struct address_space *mapping;
1733 : struct folio *folio;
1734 0 : enum folio_references references = FOLIOREF_RECLAIM;
1735 : bool dirty, writeback;
1736 : unsigned int nr_pages;
1737 :
1738 0 : cond_resched();
1739 :
1740 0 : folio = lru_to_folio(folio_list);
1741 0 : list_del(&folio->lru);
1742 :
1743 0 : if (!folio_trylock(folio))
1744 : goto keep;
1745 :
1746 : VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
1747 :
1748 0 : nr_pages = folio_nr_pages(folio);
1749 :
1750 : /* Account the number of base pages */
1751 0 : sc->nr_scanned += nr_pages;
1752 :
1753 0 : if (unlikely(!folio_evictable(folio)))
1754 : goto activate_locked;
1755 :
1756 0 : if (!sc->may_unmap && folio_mapped(folio))
1757 : goto keep_locked;
1758 :
1759 : /* folio_update_gen() tried to promote this page? */
1760 : if (lru_gen_enabled() && !ignore_references &&
1761 : folio_mapped(folio) && folio_test_referenced(folio))
1762 : goto keep_locked;
1763 :
1764 : /*
1765 : * The number of dirty pages determines if a node is marked
1766 : * reclaim_congested. kswapd will stall and start writing
1767 : * folios if the tail of the LRU is all dirty unqueued folios.
1768 : */
1769 0 : folio_check_dirty_writeback(folio, &dirty, &writeback);
1770 0 : if (dirty || writeback)
1771 0 : stat->nr_dirty += nr_pages;
1772 :
1773 0 : if (dirty && !writeback)
1774 0 : stat->nr_unqueued_dirty += nr_pages;
1775 :
1776 : /*
1777 : * Treat this folio as congested if folios are cycling
1778 : * through the LRU so quickly that the folios marked
1779 : * for immediate reclaim are making it to the end of
1780 : * the LRU a second time.
1781 : */
1782 0 : if (writeback && folio_test_reclaim(folio))
1783 0 : stat->nr_congested += nr_pages;
1784 :
1785 : /*
1786 : * If a folio at the tail of the LRU is under writeback, there
1787 : * are three cases to consider.
1788 : *
1789 : * 1) If reclaim is encountering an excessive number
1790 : * of folios under writeback and this folio has both
1791 : * the writeback and reclaim flags set, then it
1792 : * indicates that folios are being queued for I/O but
1793 : * are being recycled through the LRU before the I/O
1794 : * can complete. Waiting on the folio itself risks an
1795 : * indefinite stall if it is impossible to writeback
1796 : * the folio due to I/O error or disconnected storage
1797 : * so instead note that the LRU is being scanned too
1798 : * quickly and the caller can stall after the folio
1799 : * list has been processed.
1800 : *
1801 : * 2) Global or new memcg reclaim encounters a folio that is
1802 : * not marked for immediate reclaim, or the caller does not
1803 : * have __GFP_FS (or __GFP_IO if it's simply going to swap,
1804 : * not to fs). In this case mark the folio for immediate
1805 : * reclaim and continue scanning.
1806 : *
1807 : * Require may_enter_fs() because we would wait on fs, which
1808 : * may not have submitted I/O yet. And the loop driver might
1809 : * enter reclaim, and deadlock if it waits on a folio for
1810 : * which it is needed to do the write (loop masks off
1811 : * __GFP_IO|__GFP_FS for this reason); but more thought
1812 : * would probably show more reasons.
1813 : *
1814 : * 3) Legacy memcg encounters a folio that already has the
1815 : * reclaim flag set. memcg does not have any dirty folio
1816 : * throttling so we could easily OOM just because too many
1817 : * folios are in writeback and there is nothing else to
1818 : * reclaim. Wait for the writeback to complete.
1819 : *
1820 : * In cases 1) and 2) we activate the folios to get them out of
1821 : * the way while we continue scanning for clean folios on the
1822 : * inactive list and refilling from the active list. The
1823 : * observation here is that waiting for disk writes is more
1824 : * expensive than potentially causing reloads down the line.
1825 : * Since they're marked for immediate reclaim, they won't put
1826 : * memory pressure on the cache working set any longer than it
1827 : * takes to write them to disk.
1828 : */
1829 0 : if (folio_test_writeback(folio)) {
1830 : /* Case 1 above */
1831 0 : if (current_is_kswapd() &&
1832 0 : folio_test_reclaim(folio) &&
1833 0 : test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1834 0 : stat->nr_immediate += nr_pages;
1835 0 : goto activate_locked;
1836 :
1837 : /* Case 2 above */
1838 0 : } else if (writeback_throttling_sane(sc) ||
1839 : !folio_test_reclaim(folio) ||
1840 : !may_enter_fs(folio, sc->gfp_mask)) {
1841 : /*
1842 : * This is slightly racy -
1843 : * folio_end_writeback() might have
1844 : * just cleared the reclaim flag, then
1845 : * setting the reclaim flag here ends up
1846 : * interpreted as the readahead flag - but
1847 : * that does not matter enough to care.
1848 : * What we do want is for this folio to
1849 : * have the reclaim flag set next time
1850 : * memcg reclaim reaches the tests above,
1851 : * so it will then wait for writeback to
1852 : * avoid OOM; and it's also appropriate
1853 : * in global reclaim.
1854 : */
1855 0 : folio_set_reclaim(folio);
1856 0 : stat->nr_writeback += nr_pages;
1857 0 : goto activate_locked;
1858 :
1859 : /* Case 3 above */
1860 : } else {
1861 : folio_unlock(folio);
1862 : folio_wait_writeback(folio);
1863 : /* then go back and try same folio again */
1864 : list_add_tail(&folio->lru, folio_list);
1865 0 : continue;
1866 : }
1867 : }
1868 :
1869 0 : if (!ignore_references)
1870 0 : references = folio_check_references(folio, sc);
1871 :
1872 0 : switch (references) {
1873 : case FOLIOREF_ACTIVATE:
1874 : goto activate_locked;
1875 : case FOLIOREF_KEEP:
1876 0 : stat->nr_ref_keep += nr_pages;
1877 0 : goto keep_locked;
1878 : case FOLIOREF_RECLAIM:
1879 : case FOLIOREF_RECLAIM_CLEAN:
1880 : ; /* try to reclaim the folio below */
1881 : }
1882 :
1883 : /*
1884 : * Before reclaiming the folio, try to relocate
1885 : * its contents to another node.
1886 : */
1887 : if (do_demote_pass &&
1888 : (thp_migration_supported() || !folio_test_large(folio))) {
1889 : list_add(&folio->lru, &demote_folios);
1890 : folio_unlock(folio);
1891 : continue;
1892 : }
1893 :
1894 : /*
1895 : * Anonymous process memory has backing store?
1896 : * Try to allocate it some swap space here.
1897 : * Lazyfree folio could be freed directly
1898 : */
1899 0 : if (folio_test_anon(folio) && folio_test_swapbacked(folio)) {
1900 0 : if (!folio_test_swapcache(folio)) {
1901 0 : if (!(sc->gfp_mask & __GFP_IO))
1902 : goto keep_locked;
1903 0 : if (folio_maybe_dma_pinned(folio))
1904 : goto keep_locked;
1905 0 : if (folio_test_large(folio)) {
1906 : /* cannot split folio, skip it */
1907 : if (!can_split_folio(folio, NULL))
1908 : goto activate_locked;
1909 : /*
1910 : * Split folios without a PMD map right
1911 : * away. Chances are some or all of the
1912 : * tail pages can be freed without IO.
1913 : */
1914 : if (!folio_entire_mapcount(folio) &&
1915 : split_folio_to_list(folio,
1916 : folio_list))
1917 : goto activate_locked;
1918 : }
1919 0 : if (!add_to_swap(folio)) {
1920 0 : if (!folio_test_large(folio))
1921 : goto activate_locked_split;
1922 : /* Fallback to swap normal pages */
1923 0 : if (split_folio_to_list(folio,
1924 : folio_list))
1925 : goto activate_locked;
1926 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1927 : count_vm_event(THP_SWPOUT_FALLBACK);
1928 : #endif
1929 0 : if (!add_to_swap(folio))
1930 : goto activate_locked_split;
1931 : }
1932 : }
1933 0 : } else if (folio_test_swapbacked(folio) &&
1934 0 : folio_test_large(folio)) {
1935 : /* Split shmem folio */
1936 : if (split_folio_to_list(folio, folio_list))
1937 : goto keep_locked;
1938 : }
1939 :
1940 : /*
1941 : * If the folio was split above, the tail pages will make
1942 : * their own pass through this function and be accounted
1943 : * then.
1944 : */
1945 0 : if ((nr_pages > 1) && !folio_test_large(folio)) {
1946 0 : sc->nr_scanned -= (nr_pages - 1);
1947 0 : nr_pages = 1;
1948 : }
1949 :
1950 : /*
1951 : * The folio is mapped into the page tables of one or more
1952 : * processes. Try to unmap it here.
1953 : */
1954 0 : if (folio_mapped(folio)) {
1955 0 : enum ttu_flags flags = TTU_BATCH_FLUSH;
1956 0 : bool was_swapbacked = folio_test_swapbacked(folio);
1957 :
1958 : if (folio_test_pmd_mappable(folio))
1959 : flags |= TTU_SPLIT_HUGE_PMD;
1960 :
1961 0 : try_to_unmap(folio, flags);
1962 0 : if (folio_mapped(folio)) {
1963 0 : stat->nr_unmap_fail += nr_pages;
1964 0 : if (!was_swapbacked &&
1965 0 : folio_test_swapbacked(folio))
1966 0 : stat->nr_lazyfree_fail += nr_pages;
1967 : goto activate_locked;
1968 : }
1969 : }
1970 :
1971 : /*
1972 : * Folio is unmapped now so it cannot be newly pinned anymore.
1973 : * No point in trying to reclaim folio if it is pinned.
1974 : * Furthermore we don't want to reclaim underlying fs metadata
1975 : * if the folio is pinned and thus potentially modified by the
1976 : * pinning process as that may upset the filesystem.
1977 : */
1978 0 : if (folio_maybe_dma_pinned(folio))
1979 : goto activate_locked;
1980 :
1981 0 : mapping = folio_mapping(folio);
1982 0 : if (folio_test_dirty(folio)) {
1983 : /*
1984 : * Only kswapd can writeback filesystem folios
1985 : * to avoid risk of stack overflow. But avoid
1986 : * injecting inefficient single-folio I/O into
1987 : * flusher writeback as much as possible: only
1988 : * write folios when we've encountered many
1989 : * dirty folios, and when we've already scanned
1990 : * the rest of the LRU for clean folios and see
1991 : * the same dirty folios again (with the reclaim
1992 : * flag set).
1993 : */
1994 0 : if (folio_is_file_lru(folio) &&
1995 0 : (!current_is_kswapd() ||
1996 0 : !folio_test_reclaim(folio) ||
1997 0 : !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1998 : /*
1999 : * Immediately reclaim when written back.
2000 : * Similar in principle to folio_deactivate()
2001 : * except we already have the folio isolated
2002 : * and know it's dirty
2003 : */
2004 0 : node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE,
2005 : nr_pages);
2006 : folio_set_reclaim(folio);
2007 :
2008 : goto activate_locked;
2009 : }
2010 :
2011 0 : if (references == FOLIOREF_RECLAIM_CLEAN)
2012 : goto keep_locked;
2013 0 : if (!may_enter_fs(folio, sc->gfp_mask))
2014 : goto keep_locked;
2015 0 : if (!sc->may_writepage)
2016 : goto keep_locked;
2017 :
2018 : /*
2019 : * Folio is dirty. Flush the TLB if a writable entry
2020 : * potentially exists to avoid CPU writes after I/O
2021 : * starts and then write it out here.
2022 : */
2023 : try_to_unmap_flush_dirty();
2024 0 : switch (pageout(folio, mapping, &plug)) {
2025 : case PAGE_KEEP:
2026 : goto keep_locked;
2027 : case PAGE_ACTIVATE:
2028 : goto activate_locked;
2029 : case PAGE_SUCCESS:
2030 0 : stat->nr_pageout += nr_pages;
2031 :
2032 0 : if (folio_test_writeback(folio))
2033 : goto keep;
2034 0 : if (folio_test_dirty(folio))
2035 : goto keep;
2036 :
2037 : /*
2038 : * A synchronous write - probably a ramdisk. Go
2039 : * ahead and try to reclaim the folio.
2040 : */
2041 0 : if (!folio_trylock(folio))
2042 : goto keep;
2043 0 : if (folio_test_dirty(folio) ||
2044 0 : folio_test_writeback(folio))
2045 : goto keep_locked;
2046 0 : mapping = folio_mapping(folio);
2047 : fallthrough;
2048 : case PAGE_CLEAN:
2049 : ; /* try to free the folio below */
2050 : }
2051 : }
2052 :
2053 : /*
2054 : * If the folio has buffers, try to free the buffer
2055 : * mappings associated with this folio. If we succeed
2056 : * we try to free the folio as well.
2057 : *
2058 : * We do this even if the folio is dirty.
2059 : * filemap_release_folio() does not perform I/O, but it
2060 : * is possible for a folio to have the dirty flag set,
2061 : * but it is actually clean (all its buffers are clean).
2062 : * This happens if the buffers were written out directly,
2063 : * with submit_bh(). ext3 will do this, as well as
2064 : * the blockdev mapping. filemap_release_folio() will
2065 : * discover that cleanness and will drop the buffers
2066 : * and mark the folio clean - it can be freed.
2067 : *
2068 : * Rarely, folios can have buffers and no ->mapping.
2069 : * These are the folios which were not successfully
2070 : * invalidated in truncate_cleanup_folio(). We try to
2071 : * drop those buffers here and if that worked, and the
2072 : * folio is no longer mapped into process address space
2073 : * (refcount == 1) it can be freed. Otherwise, leave
2074 : * the folio on the LRU so it is swappable.
2075 : */
2076 0 : if (folio_has_private(folio)) {
2077 0 : if (!filemap_release_folio(folio, sc->gfp_mask))
2078 : goto activate_locked;
2079 0 : if (!mapping && folio_ref_count(folio) == 1) {
2080 0 : folio_unlock(folio);
2081 0 : if (folio_put_testzero(folio))
2082 : goto free_it;
2083 : else {
2084 : /*
2085 : * rare race with speculative reference.
2086 : * the speculative reference will free
2087 : * this folio shortly, so we may
2088 : * increment nr_reclaimed here (and
2089 : * leave it off the LRU).
2090 : */
2091 0 : nr_reclaimed += nr_pages;
2092 0 : continue;
2093 : }
2094 : }
2095 : }
2096 :
2097 0 : if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) {
2098 : /* follow __remove_mapping for reference */
2099 0 : if (!folio_ref_freeze(folio, 1))
2100 : goto keep_locked;
2101 : /*
2102 : * The folio has only one reference left, which is
2103 : * from the isolation. After the caller puts the
2104 : * folio back on the lru and drops the reference, the
2105 : * folio will be freed anyway. It doesn't matter
2106 : * which lru it goes on. So we don't bother checking
2107 : * the dirty flag here.
2108 : */
2109 0 : count_vm_events(PGLAZYFREED, nr_pages);
2110 0 : count_memcg_folio_events(folio, PGLAZYFREED, nr_pages);
2111 0 : } else if (!mapping || !__remove_mapping(mapping, folio, true,
2112 : sc->target_mem_cgroup))
2113 : goto keep_locked;
2114 :
2115 0 : folio_unlock(folio);
2116 : free_it:
2117 : /*
2118 : * Folio may get swapped out as a whole, need to account
2119 : * all pages in it.
2120 : */
2121 0 : nr_reclaimed += nr_pages;
2122 :
2123 : /*
2124 : * Is there need to periodically free_folio_list? It would
2125 : * appear not as the counts should be low
2126 : */
2127 0 : if (unlikely(folio_test_large(folio)))
2128 0 : destroy_large_folio(folio);
2129 : else
2130 0 : list_add(&folio->lru, &free_folios);
2131 0 : continue;
2132 :
2133 : activate_locked_split:
2134 : /*
2135 : * The tail pages that are failed to add into swap cache
2136 : * reach here. Fixup nr_scanned and nr_pages.
2137 : */
2138 0 : if (nr_pages > 1) {
2139 0 : sc->nr_scanned -= (nr_pages - 1);
2140 0 : nr_pages = 1;
2141 : }
2142 : activate_locked:
2143 : /* Not a candidate for swapping, so reclaim swap space. */
2144 0 : if (folio_test_swapcache(folio) &&
2145 0 : (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio)))
2146 0 : folio_free_swap(folio);
2147 : VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
2148 0 : if (!folio_test_mlocked(folio)) {
2149 0 : int type = folio_is_file_lru(folio);
2150 0 : folio_set_active(folio);
2151 0 : stat->nr_activate[type] += nr_pages;
2152 0 : count_memcg_folio_events(folio, PGACTIVATE, nr_pages);
2153 : }
2154 : keep_locked:
2155 0 : folio_unlock(folio);
2156 : keep:
2157 0 : list_add(&folio->lru, &ret_folios);
2158 : VM_BUG_ON_FOLIO(folio_test_lru(folio) ||
2159 : folio_test_unevictable(folio), folio);
2160 : }
2161 : /* 'folio_list' is always empty here */
2162 :
2163 : /* Migrate folios selected for demotion */
2164 0 : nr_reclaimed += demote_folio_list(&demote_folios, pgdat);
2165 : /* Folios that could not be demoted are still in @demote_folios */
2166 0 : if (!list_empty(&demote_folios)) {
2167 : /* Folios which weren't demoted go back on @folio_list */
2168 0 : list_splice_init(&demote_folios, folio_list);
2169 :
2170 : /*
2171 : * goto retry to reclaim the undemoted folios in folio_list if
2172 : * desired.
2173 : *
2174 : * Reclaiming directly from top tier nodes is not often desired
2175 : * due to it breaking the LRU ordering: in general memory
2176 : * should be reclaimed from lower tier nodes and demoted from
2177 : * top tier nodes.
2178 : *
2179 : * However, disabling reclaim from top tier nodes entirely
2180 : * would cause ooms in edge scenarios where lower tier memory
2181 : * is unreclaimable for whatever reason, eg memory being
2182 : * mlocked or too hot to reclaim. We can disable reclaim
2183 : * from top tier nodes in proactive reclaim though as that is
2184 : * not real memory pressure.
2185 : */
2186 0 : if (!sc->proactive) {
2187 : do_demote_pass = false;
2188 : goto retry;
2189 : }
2190 : }
2191 :
2192 0 : pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
2193 :
2194 0 : mem_cgroup_uncharge_list(&free_folios);
2195 : try_to_unmap_flush();
2196 0 : free_unref_page_list(&free_folios);
2197 :
2198 0 : list_splice(&ret_folios, folio_list);
2199 0 : count_vm_events(PGACTIVATE, pgactivate);
2200 :
2201 0 : if (plug)
2202 0 : swap_write_unplug(plug);
2203 0 : return nr_reclaimed;
2204 : }
2205 :
2206 0 : unsigned int reclaim_clean_pages_from_list(struct zone *zone,
2207 : struct list_head *folio_list)
2208 : {
2209 0 : struct scan_control sc = {
2210 : .gfp_mask = GFP_KERNEL,
2211 : .may_unmap = 1,
2212 : };
2213 : struct reclaim_stat stat;
2214 : unsigned int nr_reclaimed;
2215 : struct folio *folio, *next;
2216 0 : LIST_HEAD(clean_folios);
2217 : unsigned int noreclaim_flag;
2218 :
2219 0 : list_for_each_entry_safe(folio, next, folio_list, lru) {
2220 0 : if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) &&
2221 0 : !folio_test_dirty(folio) && !__folio_test_movable(folio) &&
2222 0 : !folio_test_unevictable(folio)) {
2223 0 : folio_clear_active(folio);
2224 0 : list_move(&folio->lru, &clean_folios);
2225 : }
2226 : }
2227 :
2228 : /*
2229 : * We should be safe here since we are only dealing with file pages and
2230 : * we are not kswapd and therefore cannot write dirty file pages. But
2231 : * call memalloc_noreclaim_save() anyway, just in case these conditions
2232 : * change in the future.
2233 : */
2234 0 : noreclaim_flag = memalloc_noreclaim_save();
2235 0 : nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc,
2236 : &stat, true);
2237 0 : memalloc_noreclaim_restore(noreclaim_flag);
2238 :
2239 0 : list_splice(&clean_folios, folio_list);
2240 0 : mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
2241 : -(long)nr_reclaimed);
2242 : /*
2243 : * Since lazyfree pages are isolated from file LRU from the beginning,
2244 : * they will rotate back to anonymous LRU in the end if it failed to
2245 : * discard so isolated count will be mismatched.
2246 : * Compensate the isolated count for both LRU lists.
2247 : */
2248 0 : mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
2249 0 : stat.nr_lazyfree_fail);
2250 0 : mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
2251 0 : -(long)stat.nr_lazyfree_fail);
2252 0 : return nr_reclaimed;
2253 : }
2254 :
2255 : /*
2256 : * Update LRU sizes after isolating pages. The LRU size updates must
2257 : * be complete before mem_cgroup_update_lru_size due to a sanity check.
2258 : */
2259 : static __always_inline void update_lru_sizes(struct lruvec *lruvec,
2260 : enum lru_list lru, unsigned long *nr_zone_taken)
2261 : {
2262 : int zid;
2263 :
2264 0 : for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2265 0 : if (!nr_zone_taken[zid])
2266 0 : continue;
2267 :
2268 0 : update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
2269 : }
2270 :
2271 : }
2272 :
2273 : /*
2274 : * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
2275 : *
2276 : * lruvec->lru_lock is heavily contended. Some of the functions that
2277 : * shrink the lists perform better by taking out a batch of pages
2278 : * and working on them outside the LRU lock.
2279 : *
2280 : * For pagecache intensive workloads, this function is the hottest
2281 : * spot in the kernel (apart from copy_*_user functions).
2282 : *
2283 : * Lru_lock must be held before calling this function.
2284 : *
2285 : * @nr_to_scan: The number of eligible pages to look through on the list.
2286 : * @lruvec: The LRU vector to pull pages from.
2287 : * @dst: The temp list to put pages on to.
2288 : * @nr_scanned: The number of pages that were scanned.
2289 : * @sc: The scan_control struct for this reclaim session
2290 : * @lru: LRU list id for isolating
2291 : *
2292 : * returns how many pages were moved onto *@dst.
2293 : */
2294 0 : static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
2295 : struct lruvec *lruvec, struct list_head *dst,
2296 : unsigned long *nr_scanned, struct scan_control *sc,
2297 : enum lru_list lru)
2298 : {
2299 0 : struct list_head *src = &lruvec->lists[lru];
2300 0 : unsigned long nr_taken = 0;
2301 0 : unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
2302 0 : unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
2303 0 : unsigned long skipped = 0;
2304 : unsigned long scan, total_scan, nr_pages;
2305 0 : LIST_HEAD(folios_skipped);
2306 :
2307 0 : total_scan = 0;
2308 0 : scan = 0;
2309 0 : while (scan < nr_to_scan && !list_empty(src)) {
2310 0 : struct list_head *move_to = src;
2311 : struct folio *folio;
2312 :
2313 0 : folio = lru_to_folio(src);
2314 : prefetchw_prev_lru_folio(folio, src, flags);
2315 :
2316 0 : nr_pages = folio_nr_pages(folio);
2317 0 : total_scan += nr_pages;
2318 :
2319 0 : if (folio_zonenum(folio) > sc->reclaim_idx) {
2320 0 : nr_skipped[folio_zonenum(folio)] += nr_pages;
2321 0 : move_to = &folios_skipped;
2322 0 : goto move;
2323 : }
2324 :
2325 : /*
2326 : * Do not count skipped folios because that makes the function
2327 : * return with no isolated folios if the LRU mostly contains
2328 : * ineligible folios. This causes the VM to not reclaim any
2329 : * folios, triggering a premature OOM.
2330 : * Account all pages in a folio.
2331 : */
2332 0 : scan += nr_pages;
2333 :
2334 0 : if (!folio_test_lru(folio))
2335 : goto move;
2336 0 : if (!sc->may_unmap && folio_mapped(folio))
2337 : goto move;
2338 :
2339 : /*
2340 : * Be careful not to clear the lru flag until after we're
2341 : * sure the folio is not being freed elsewhere -- the
2342 : * folio release code relies on it.
2343 : */
2344 0 : if (unlikely(!folio_try_get(folio)))
2345 : goto move;
2346 :
2347 0 : if (!folio_test_clear_lru(folio)) {
2348 : /* Another thread is already isolating this folio */
2349 : folio_put(folio);
2350 : goto move;
2351 : }
2352 :
2353 0 : nr_taken += nr_pages;
2354 0 : nr_zone_taken[folio_zonenum(folio)] += nr_pages;
2355 0 : move_to = dst;
2356 : move:
2357 0 : list_move(&folio->lru, move_to);
2358 : }
2359 :
2360 : /*
2361 : * Splice any skipped folios to the start of the LRU list. Note that
2362 : * this disrupts the LRU order when reclaiming for lower zones but
2363 : * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
2364 : * scanning would soon rescan the same folios to skip and waste lots
2365 : * of cpu cycles.
2366 : */
2367 0 : if (!list_empty(&folios_skipped)) {
2368 : int zid;
2369 :
2370 : list_splice(&folios_skipped, src);
2371 0 : for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2372 0 : if (!nr_skipped[zid])
2373 0 : continue;
2374 :
2375 0 : __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
2376 0 : skipped += nr_skipped[zid];
2377 : }
2378 : }
2379 0 : *nr_scanned = total_scan;
2380 0 : trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
2381 : total_scan, skipped, nr_taken,
2382 : sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru);
2383 0 : update_lru_sizes(lruvec, lru, nr_zone_taken);
2384 0 : return nr_taken;
2385 : }
2386 :
2387 : /**
2388 : * folio_isolate_lru() - Try to isolate a folio from its LRU list.
2389 : * @folio: Folio to isolate from its LRU list.
2390 : *
2391 : * Isolate a @folio from an LRU list and adjust the vmstat statistic
2392 : * corresponding to whatever LRU list the folio was on.
2393 : *
2394 : * The folio will have its LRU flag cleared. If it was found on the
2395 : * active list, it will have the Active flag set. If it was found on the
2396 : * unevictable list, it will have the Unevictable flag set. These flags
2397 : * may need to be cleared by the caller before letting the page go.
2398 : *
2399 : * Context:
2400 : *
2401 : * (1) Must be called with an elevated refcount on the folio. This is a
2402 : * fundamental difference from isolate_lru_folios() (which is called
2403 : * without a stable reference).
2404 : * (2) The lru_lock must not be held.
2405 : * (3) Interrupts must be enabled.
2406 : *
2407 : * Return: true if the folio was removed from an LRU list.
2408 : * false if the folio was not on an LRU list.
2409 : */
2410 0 : bool folio_isolate_lru(struct folio *folio)
2411 : {
2412 0 : bool ret = false;
2413 :
2414 : VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio);
2415 :
2416 0 : if (folio_test_clear_lru(folio)) {
2417 : struct lruvec *lruvec;
2418 :
2419 0 : folio_get(folio);
2420 0 : lruvec = folio_lruvec_lock_irq(folio);
2421 0 : lruvec_del_folio(lruvec, folio);
2422 : unlock_page_lruvec_irq(lruvec);
2423 0 : ret = true;
2424 : }
2425 :
2426 0 : return ret;
2427 : }
2428 :
2429 : /*
2430 : * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
2431 : * then get rescheduled. When there are massive number of tasks doing page
2432 : * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
2433 : * the LRU list will go small and be scanned faster than necessary, leading to
2434 : * unnecessary swapping, thrashing and OOM.
2435 : */
2436 0 : static int too_many_isolated(struct pglist_data *pgdat, int file,
2437 : struct scan_control *sc)
2438 : {
2439 : unsigned long inactive, isolated;
2440 : bool too_many;
2441 :
2442 0 : if (current_is_kswapd())
2443 : return 0;
2444 :
2445 0 : if (!writeback_throttling_sane(sc))
2446 : return 0;
2447 :
2448 0 : if (file) {
2449 0 : inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
2450 0 : isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
2451 : } else {
2452 0 : inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
2453 0 : isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
2454 : }
2455 :
2456 : /*
2457 : * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
2458 : * won't get blocked by normal direct-reclaimers, forming a circular
2459 : * deadlock.
2460 : */
2461 0 : if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
2462 0 : inactive >>= 3;
2463 :
2464 0 : too_many = isolated > inactive;
2465 :
2466 : /* Wake up tasks throttled due to too_many_isolated. */
2467 0 : if (!too_many)
2468 : wake_throttle_isolated(pgdat);
2469 :
2470 0 : return too_many;
2471 : }
2472 :
2473 : /*
2474 : * move_folios_to_lru() moves folios from private @list to appropriate LRU list.
2475 : * On return, @list is reused as a list of folios to be freed by the caller.
2476 : *
2477 : * Returns the number of pages moved to the given lruvec.
2478 : */
2479 0 : static unsigned int move_folios_to_lru(struct lruvec *lruvec,
2480 : struct list_head *list)
2481 : {
2482 0 : int nr_pages, nr_moved = 0;
2483 0 : LIST_HEAD(folios_to_free);
2484 :
2485 0 : while (!list_empty(list)) {
2486 0 : struct folio *folio = lru_to_folio(list);
2487 :
2488 : VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
2489 0 : list_del(&folio->lru);
2490 0 : if (unlikely(!folio_evictable(folio))) {
2491 0 : spin_unlock_irq(&lruvec->lru_lock);
2492 0 : folio_putback_lru(folio);
2493 0 : spin_lock_irq(&lruvec->lru_lock);
2494 0 : continue;
2495 : }
2496 :
2497 : /*
2498 : * The folio_set_lru needs to be kept here for list integrity.
2499 : * Otherwise:
2500 : * #0 move_folios_to_lru #1 release_pages
2501 : * if (!folio_put_testzero())
2502 : * if (folio_put_testzero())
2503 : * !lru //skip lru_lock
2504 : * folio_set_lru()
2505 : * list_add(&folio->lru,)
2506 : * list_add(&folio->lru,)
2507 : */
2508 0 : folio_set_lru(folio);
2509 :
2510 0 : if (unlikely(folio_put_testzero(folio))) {
2511 0 : __folio_clear_lru_flags(folio);
2512 :
2513 0 : if (unlikely(folio_test_large(folio))) {
2514 0 : spin_unlock_irq(&lruvec->lru_lock);
2515 0 : destroy_large_folio(folio);
2516 0 : spin_lock_irq(&lruvec->lru_lock);
2517 : } else
2518 0 : list_add(&folio->lru, &folios_to_free);
2519 :
2520 0 : continue;
2521 : }
2522 :
2523 : /*
2524 : * All pages were isolated from the same lruvec (and isolation
2525 : * inhibits memcg migration).
2526 : */
2527 : VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
2528 0 : lruvec_add_folio(lruvec, folio);
2529 0 : nr_pages = folio_nr_pages(folio);
2530 0 : nr_moved += nr_pages;
2531 0 : if (folio_test_active(folio))
2532 0 : workingset_age_nonresident(lruvec, nr_pages);
2533 : }
2534 :
2535 : /*
2536 : * To save our caller's stack, now use input list for pages to free.
2537 : */
2538 0 : list_splice(&folios_to_free, list);
2539 :
2540 0 : return nr_moved;
2541 : }
2542 :
2543 : /*
2544 : * If a kernel thread (such as nfsd for loop-back mounts) services a backing
2545 : * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case
2546 : * we should not throttle. Otherwise it is safe to do so.
2547 : */
2548 : static int current_may_throttle(void)
2549 : {
2550 0 : return !(current->flags & PF_LOCAL_THROTTLE);
2551 : }
2552 :
2553 : /*
2554 : * shrink_inactive_list() is a helper for shrink_node(). It returns the number
2555 : * of reclaimed pages
2556 : */
2557 0 : static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
2558 : struct lruvec *lruvec, struct scan_control *sc,
2559 : enum lru_list lru)
2560 : {
2561 0 : LIST_HEAD(folio_list);
2562 : unsigned long nr_scanned;
2563 0 : unsigned int nr_reclaimed = 0;
2564 : unsigned long nr_taken;
2565 : struct reclaim_stat stat;
2566 0 : bool file = is_file_lru(lru);
2567 : enum vm_event_item item;
2568 0 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2569 0 : bool stalled = false;
2570 :
2571 0 : while (unlikely(too_many_isolated(pgdat, file, sc))) {
2572 0 : if (stalled)
2573 : return 0;
2574 :
2575 : /* wait a bit for the reclaimer. */
2576 0 : stalled = true;
2577 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
2578 :
2579 : /* We are about to die and free our memory. Return now. */
2580 0 : if (fatal_signal_pending(current))
2581 : return SWAP_CLUSTER_MAX;
2582 : }
2583 :
2584 0 : lru_add_drain();
2585 :
2586 0 : spin_lock_irq(&lruvec->lru_lock);
2587 :
2588 0 : nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list,
2589 : &nr_scanned, sc, lru);
2590 :
2591 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2592 0 : item = PGSCAN_KSWAPD + reclaimer_offset();
2593 0 : if (!cgroup_reclaim(sc))
2594 0 : __count_vm_events(item, nr_scanned);
2595 0 : __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
2596 0 : __count_vm_events(PGSCAN_ANON + file, nr_scanned);
2597 :
2598 0 : spin_unlock_irq(&lruvec->lru_lock);
2599 :
2600 0 : if (nr_taken == 0)
2601 : return 0;
2602 :
2603 0 : nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false);
2604 :
2605 0 : spin_lock_irq(&lruvec->lru_lock);
2606 0 : move_folios_to_lru(lruvec, &folio_list);
2607 :
2608 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2609 0 : item = PGSTEAL_KSWAPD + reclaimer_offset();
2610 0 : if (!cgroup_reclaim(sc))
2611 0 : __count_vm_events(item, nr_reclaimed);
2612 0 : __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
2613 0 : __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);
2614 0 : spin_unlock_irq(&lruvec->lru_lock);
2615 :
2616 0 : lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed);
2617 0 : mem_cgroup_uncharge_list(&folio_list);
2618 0 : free_unref_page_list(&folio_list);
2619 :
2620 : /*
2621 : * If dirty folios are scanned that are not queued for IO, it
2622 : * implies that flushers are not doing their job. This can
2623 : * happen when memory pressure pushes dirty folios to the end of
2624 : * the LRU before the dirty limits are breached and the dirty
2625 : * data has expired. It can also happen when the proportion of
2626 : * dirty folios grows not through writes but through memory
2627 : * pressure reclaiming all the clean cache. And in some cases,
2628 : * the flushers simply cannot keep up with the allocation
2629 : * rate. Nudge the flusher threads in case they are asleep.
2630 : */
2631 0 : if (stat.nr_unqueued_dirty == nr_taken) {
2632 0 : wakeup_flusher_threads(WB_REASON_VMSCAN);
2633 : /*
2634 : * For cgroupv1 dirty throttling is achieved by waking up
2635 : * the kernel flusher here and later waiting on folios
2636 : * which are in writeback to finish (see shrink_folio_list()).
2637 : *
2638 : * Flusher may not be able to issue writeback quickly
2639 : * enough for cgroupv1 writeback throttling to work
2640 : * on a large system.
2641 : */
2642 0 : if (!writeback_throttling_sane(sc))
2643 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
2644 : }
2645 :
2646 0 : sc->nr.dirty += stat.nr_dirty;
2647 0 : sc->nr.congested += stat.nr_congested;
2648 0 : sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
2649 0 : sc->nr.writeback += stat.nr_writeback;
2650 0 : sc->nr.immediate += stat.nr_immediate;
2651 0 : sc->nr.taken += nr_taken;
2652 0 : if (file)
2653 0 : sc->nr.file_taken += nr_taken;
2654 :
2655 0 : trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2656 0 : nr_scanned, nr_reclaimed, &stat, sc->priority, file);
2657 0 : return nr_reclaimed;
2658 : }
2659 :
2660 : /*
2661 : * shrink_active_list() moves folios from the active LRU to the inactive LRU.
2662 : *
2663 : * We move them the other way if the folio is referenced by one or more
2664 : * processes.
2665 : *
2666 : * If the folios are mostly unmapped, the processing is fast and it is
2667 : * appropriate to hold lru_lock across the whole operation. But if
2668 : * the folios are mapped, the processing is slow (folio_referenced()), so
2669 : * we should drop lru_lock around each folio. It's impossible to balance
2670 : * this, so instead we remove the folios from the LRU while processing them.
2671 : * It is safe to rely on the active flag against the non-LRU folios in here
2672 : * because nobody will play with that bit on a non-LRU folio.
2673 : *
2674 : * The downside is that we have to touch folio->_refcount against each folio.
2675 : * But we had to alter folio->flags anyway.
2676 : */
2677 0 : static void shrink_active_list(unsigned long nr_to_scan,
2678 : struct lruvec *lruvec,
2679 : struct scan_control *sc,
2680 : enum lru_list lru)
2681 : {
2682 : unsigned long nr_taken;
2683 : unsigned long nr_scanned;
2684 : unsigned long vm_flags;
2685 0 : LIST_HEAD(l_hold); /* The folios which were snipped off */
2686 0 : LIST_HEAD(l_active);
2687 0 : LIST_HEAD(l_inactive);
2688 : unsigned nr_deactivate, nr_activate;
2689 0 : unsigned nr_rotated = 0;
2690 0 : int file = is_file_lru(lru);
2691 0 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2692 :
2693 0 : lru_add_drain();
2694 :
2695 0 : spin_lock_irq(&lruvec->lru_lock);
2696 :
2697 0 : nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold,
2698 : &nr_scanned, sc, lru);
2699 :
2700 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2701 :
2702 0 : if (!cgroup_reclaim(sc))
2703 0 : __count_vm_events(PGREFILL, nr_scanned);
2704 0 : __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2705 :
2706 0 : spin_unlock_irq(&lruvec->lru_lock);
2707 :
2708 0 : while (!list_empty(&l_hold)) {
2709 : struct folio *folio;
2710 :
2711 0 : cond_resched();
2712 0 : folio = lru_to_folio(&l_hold);
2713 0 : list_del(&folio->lru);
2714 :
2715 0 : if (unlikely(!folio_evictable(folio))) {
2716 0 : folio_putback_lru(folio);
2717 0 : continue;
2718 : }
2719 :
2720 0 : if (unlikely(buffer_heads_over_limit)) {
2721 0 : if (folio_test_private(folio) && folio_trylock(folio)) {
2722 0 : if (folio_test_private(folio))
2723 0 : filemap_release_folio(folio, 0);
2724 0 : folio_unlock(folio);
2725 : }
2726 : }
2727 :
2728 : /* Referenced or rmap lock contention: rotate */
2729 0 : if (folio_referenced(folio, 0, sc->target_mem_cgroup,
2730 : &vm_flags) != 0) {
2731 : /*
2732 : * Identify referenced, file-backed active folios and
2733 : * give them one more trip around the active list. So
2734 : * that executable code get better chances to stay in
2735 : * memory under moderate memory pressure. Anon folios
2736 : * are not likely to be evicted by use-once streaming
2737 : * IO, plus JVM can create lots of anon VM_EXEC folios,
2738 : * so we ignore them here.
2739 : */
2740 0 : if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) {
2741 0 : nr_rotated += folio_nr_pages(folio);
2742 0 : list_add(&folio->lru, &l_active);
2743 0 : continue;
2744 : }
2745 : }
2746 :
2747 0 : folio_clear_active(folio); /* we are de-activating */
2748 0 : folio_set_workingset(folio);
2749 0 : list_add(&folio->lru, &l_inactive);
2750 : }
2751 :
2752 : /*
2753 : * Move folios back to the lru list.
2754 : */
2755 0 : spin_lock_irq(&lruvec->lru_lock);
2756 :
2757 0 : nr_activate = move_folios_to_lru(lruvec, &l_active);
2758 0 : nr_deactivate = move_folios_to_lru(lruvec, &l_inactive);
2759 : /* Keep all free folios in l_active list */
2760 0 : list_splice(&l_inactive, &l_active);
2761 :
2762 0 : __count_vm_events(PGDEACTIVATE, nr_deactivate);
2763 0 : __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
2764 :
2765 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2766 0 : spin_unlock_irq(&lruvec->lru_lock);
2767 :
2768 0 : if (nr_rotated)
2769 0 : lru_note_cost(lruvec, file, 0, nr_rotated);
2770 0 : mem_cgroup_uncharge_list(&l_active);
2771 0 : free_unref_page_list(&l_active);
2772 0 : trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2773 0 : nr_deactivate, nr_rotated, sc->priority, file);
2774 0 : }
2775 :
2776 0 : static unsigned int reclaim_folio_list(struct list_head *folio_list,
2777 : struct pglist_data *pgdat)
2778 : {
2779 : struct reclaim_stat dummy_stat;
2780 : unsigned int nr_reclaimed;
2781 : struct folio *folio;
2782 0 : struct scan_control sc = {
2783 : .gfp_mask = GFP_KERNEL,
2784 : .may_writepage = 1,
2785 : .may_unmap = 1,
2786 : .may_swap = 1,
2787 : .no_demotion = 1,
2788 : };
2789 :
2790 0 : nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false);
2791 0 : while (!list_empty(folio_list)) {
2792 0 : folio = lru_to_folio(folio_list);
2793 0 : list_del(&folio->lru);
2794 0 : folio_putback_lru(folio);
2795 : }
2796 :
2797 0 : return nr_reclaimed;
2798 : }
2799 :
2800 0 : unsigned long reclaim_pages(struct list_head *folio_list)
2801 : {
2802 : int nid;
2803 0 : unsigned int nr_reclaimed = 0;
2804 0 : LIST_HEAD(node_folio_list);
2805 : unsigned int noreclaim_flag;
2806 :
2807 0 : if (list_empty(folio_list))
2808 : return nr_reclaimed;
2809 :
2810 0 : noreclaim_flag = memalloc_noreclaim_save();
2811 :
2812 0 : nid = folio_nid(lru_to_folio(folio_list));
2813 : do {
2814 0 : struct folio *folio = lru_to_folio(folio_list);
2815 :
2816 : if (nid == folio_nid(folio)) {
2817 0 : folio_clear_active(folio);
2818 0 : list_move(&folio->lru, &node_folio_list);
2819 0 : continue;
2820 : }
2821 :
2822 : nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
2823 : nid = folio_nid(lru_to_folio(folio_list));
2824 0 : } while (!list_empty(folio_list));
2825 :
2826 0 : nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
2827 :
2828 0 : memalloc_noreclaim_restore(noreclaim_flag);
2829 :
2830 0 : return nr_reclaimed;
2831 : }
2832 :
2833 0 : static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2834 : struct lruvec *lruvec, struct scan_control *sc)
2835 : {
2836 0 : if (is_active_lru(lru)) {
2837 0 : if (sc->may_deactivate & (1 << is_file_lru(lru)))
2838 0 : shrink_active_list(nr_to_scan, lruvec, sc, lru);
2839 : else
2840 0 : sc->skipped_deactivate = 1;
2841 : return 0;
2842 : }
2843 :
2844 0 : return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2845 : }
2846 :
2847 : /*
2848 : * The inactive anon list should be small enough that the VM never has
2849 : * to do too much work.
2850 : *
2851 : * The inactive file list should be small enough to leave most memory
2852 : * to the established workingset on the scan-resistant active list,
2853 : * but large enough to avoid thrashing the aggregate readahead window.
2854 : *
2855 : * Both inactive lists should also be large enough that each inactive
2856 : * folio has a chance to be referenced again before it is reclaimed.
2857 : *
2858 : * If that fails and refaulting is observed, the inactive list grows.
2859 : *
2860 : * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios
2861 : * on this LRU, maintained by the pageout code. An inactive_ratio
2862 : * of 3 means 3:1 or 25% of the folios are kept on the inactive list.
2863 : *
2864 : * total target max
2865 : * memory ratio inactive
2866 : * -------------------------------------
2867 : * 10MB 1 5MB
2868 : * 100MB 1 50MB
2869 : * 1GB 3 250MB
2870 : * 10GB 10 0.9GB
2871 : * 100GB 31 3GB
2872 : * 1TB 101 10GB
2873 : * 10TB 320 32GB
2874 : */
2875 0 : static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2876 : {
2877 0 : enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2878 : unsigned long inactive, active;
2879 : unsigned long inactive_ratio;
2880 : unsigned long gb;
2881 :
2882 0 : inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2883 0 : active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2884 :
2885 0 : gb = (inactive + active) >> (30 - PAGE_SHIFT);
2886 0 : if (gb)
2887 0 : inactive_ratio = int_sqrt(10 * gb);
2888 : else
2889 : inactive_ratio = 1;
2890 :
2891 0 : return inactive * inactive_ratio < active;
2892 : }
2893 :
2894 : enum scan_balance {
2895 : SCAN_EQUAL,
2896 : SCAN_FRACT,
2897 : SCAN_ANON,
2898 : SCAN_FILE,
2899 : };
2900 :
2901 0 : static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc)
2902 : {
2903 : unsigned long file;
2904 : struct lruvec *target_lruvec;
2905 :
2906 : if (lru_gen_enabled())
2907 : return;
2908 :
2909 0 : target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
2910 :
2911 : /*
2912 : * Flush the memory cgroup stats, so that we read accurate per-memcg
2913 : * lruvec stats for heuristics.
2914 : */
2915 : mem_cgroup_flush_stats();
2916 :
2917 : /*
2918 : * Determine the scan balance between anon and file LRUs.
2919 : */
2920 0 : spin_lock_irq(&target_lruvec->lru_lock);
2921 0 : sc->anon_cost = target_lruvec->anon_cost;
2922 0 : sc->file_cost = target_lruvec->file_cost;
2923 0 : spin_unlock_irq(&target_lruvec->lru_lock);
2924 :
2925 : /*
2926 : * Target desirable inactive:active list ratios for the anon
2927 : * and file LRU lists.
2928 : */
2929 0 : if (!sc->force_deactivate) {
2930 : unsigned long refaults;
2931 :
2932 : /*
2933 : * When refaults are being observed, it means a new
2934 : * workingset is being established. Deactivate to get
2935 : * rid of any stale active pages quickly.
2936 : */
2937 0 : refaults = lruvec_page_state(target_lruvec,
2938 : WORKINGSET_ACTIVATE_ANON);
2939 0 : if (refaults != target_lruvec->refaults[WORKINGSET_ANON] ||
2940 0 : inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
2941 0 : sc->may_deactivate |= DEACTIVATE_ANON;
2942 : else
2943 0 : sc->may_deactivate &= ~DEACTIVATE_ANON;
2944 :
2945 0 : refaults = lruvec_page_state(target_lruvec,
2946 : WORKINGSET_ACTIVATE_FILE);
2947 0 : if (refaults != target_lruvec->refaults[WORKINGSET_FILE] ||
2948 0 : inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
2949 0 : sc->may_deactivate |= DEACTIVATE_FILE;
2950 : else
2951 0 : sc->may_deactivate &= ~DEACTIVATE_FILE;
2952 : } else
2953 0 : sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
2954 :
2955 : /*
2956 : * If we have plenty of inactive file pages that aren't
2957 : * thrashing, try to reclaim those first before touching
2958 : * anonymous pages.
2959 : */
2960 0 : file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
2961 0 : if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
2962 0 : sc->cache_trim_mode = 1;
2963 : else
2964 0 : sc->cache_trim_mode = 0;
2965 :
2966 : /*
2967 : * Prevent the reclaimer from falling into the cache trap: as
2968 : * cache pages start out inactive, every cache fault will tip
2969 : * the scan balance towards the file LRU. And as the file LRU
2970 : * shrinks, so does the window for rotation from references.
2971 : * This means we have a runaway feedback loop where a tiny
2972 : * thrashing file LRU becomes infinitely more attractive than
2973 : * anon pages. Try to detect this based on file LRU size.
2974 : */
2975 0 : if (!cgroup_reclaim(sc)) {
2976 0 : unsigned long total_high_wmark = 0;
2977 : unsigned long free, anon;
2978 : int z;
2979 :
2980 0 : free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2981 0 : file = node_page_state(pgdat, NR_ACTIVE_FILE) +
2982 0 : node_page_state(pgdat, NR_INACTIVE_FILE);
2983 :
2984 0 : for (z = 0; z < MAX_NR_ZONES; z++) {
2985 0 : struct zone *zone = &pgdat->node_zones[z];
2986 :
2987 0 : if (!managed_zone(zone))
2988 0 : continue;
2989 :
2990 0 : total_high_wmark += high_wmark_pages(zone);
2991 : }
2992 :
2993 : /*
2994 : * Consider anon: if that's low too, this isn't a
2995 : * runaway file reclaim problem, but rather just
2996 : * extreme pressure. Reclaim as per usual then.
2997 : */
2998 0 : anon = node_page_state(pgdat, NR_INACTIVE_ANON);
2999 :
3000 0 : sc->file_is_tiny =
3001 0 : file + free <= total_high_wmark &&
3002 0 : !(sc->may_deactivate & DEACTIVATE_ANON) &&
3003 0 : anon >> sc->priority;
3004 : }
3005 : }
3006 :
3007 : /*
3008 : * Determine how aggressively the anon and file LRU lists should be
3009 : * scanned.
3010 : *
3011 : * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan
3012 : * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan
3013 : */
3014 0 : static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
3015 : unsigned long *nr)
3016 : {
3017 0 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
3018 0 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3019 : unsigned long anon_cost, file_cost, total_cost;
3020 0 : int swappiness = mem_cgroup_swappiness(memcg);
3021 : u64 fraction[ANON_AND_FILE];
3022 0 : u64 denominator = 0; /* gcc */
3023 : enum scan_balance scan_balance;
3024 : unsigned long ap, fp;
3025 : enum lru_list lru;
3026 :
3027 : /* If we have no swap space, do not bother scanning anon folios. */
3028 0 : if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
3029 : scan_balance = SCAN_FILE;
3030 : goto out;
3031 : }
3032 :
3033 : /*
3034 : * Global reclaim will swap to prevent OOM even with no
3035 : * swappiness, but memcg users want to use this knob to
3036 : * disable swapping for individual groups completely when
3037 : * using the memory controller's swap limit feature would be
3038 : * too expensive.
3039 : */
3040 0 : if (cgroup_reclaim(sc) && !swappiness) {
3041 : scan_balance = SCAN_FILE;
3042 : goto out;
3043 : }
3044 :
3045 : /*
3046 : * Do not apply any pressure balancing cleverness when the
3047 : * system is close to OOM, scan both anon and file equally
3048 : * (unless the swappiness setting disagrees with swapping).
3049 : */
3050 0 : if (!sc->priority && swappiness) {
3051 : scan_balance = SCAN_EQUAL;
3052 : goto out;
3053 : }
3054 :
3055 : /*
3056 : * If the system is almost out of file pages, force-scan anon.
3057 : */
3058 0 : if (sc->file_is_tiny) {
3059 : scan_balance = SCAN_ANON;
3060 : goto out;
3061 : }
3062 :
3063 : /*
3064 : * If there is enough inactive page cache, we do not reclaim
3065 : * anything from the anonymous working right now.
3066 : */
3067 0 : if (sc->cache_trim_mode) {
3068 : scan_balance = SCAN_FILE;
3069 : goto out;
3070 : }
3071 :
3072 0 : scan_balance = SCAN_FRACT;
3073 : /*
3074 : * Calculate the pressure balance between anon and file pages.
3075 : *
3076 : * The amount of pressure we put on each LRU is inversely
3077 : * proportional to the cost of reclaiming each list, as
3078 : * determined by the share of pages that are refaulting, times
3079 : * the relative IO cost of bringing back a swapped out
3080 : * anonymous page vs reloading a filesystem page (swappiness).
3081 : *
3082 : * Although we limit that influence to ensure no list gets
3083 : * left behind completely: at least a third of the pressure is
3084 : * applied, before swappiness.
3085 : *
3086 : * With swappiness at 100, anon and file have equal IO cost.
3087 : */
3088 0 : total_cost = sc->anon_cost + sc->file_cost;
3089 0 : anon_cost = total_cost + sc->anon_cost;
3090 0 : file_cost = total_cost + sc->file_cost;
3091 0 : total_cost = anon_cost + file_cost;
3092 :
3093 0 : ap = swappiness * (total_cost + 1);
3094 0 : ap /= anon_cost + 1;
3095 :
3096 0 : fp = (200 - swappiness) * (total_cost + 1);
3097 0 : fp /= file_cost + 1;
3098 :
3099 0 : fraction[0] = ap;
3100 0 : fraction[1] = fp;
3101 0 : denominator = ap + fp;
3102 : out:
3103 0 : for_each_evictable_lru(lru) {
3104 0 : int file = is_file_lru(lru);
3105 : unsigned long lruvec_size;
3106 : unsigned long low, min;
3107 : unsigned long scan;
3108 :
3109 0 : lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
3110 0 : mem_cgroup_protection(sc->target_mem_cgroup, memcg,
3111 : &min, &low);
3112 :
3113 : if (min || low) {
3114 : /*
3115 : * Scale a cgroup's reclaim pressure by proportioning
3116 : * its current usage to its memory.low or memory.min
3117 : * setting.
3118 : *
3119 : * This is important, as otherwise scanning aggression
3120 : * becomes extremely binary -- from nothing as we
3121 : * approach the memory protection threshold, to totally
3122 : * nominal as we exceed it. This results in requiring
3123 : * setting extremely liberal protection thresholds. It
3124 : * also means we simply get no protection at all if we
3125 : * set it too low, which is not ideal.
3126 : *
3127 : * If there is any protection in place, we reduce scan
3128 : * pressure by how much of the total memory used is
3129 : * within protection thresholds.
3130 : *
3131 : * There is one special case: in the first reclaim pass,
3132 : * we skip over all groups that are within their low
3133 : * protection. If that fails to reclaim enough pages to
3134 : * satisfy the reclaim goal, we come back and override
3135 : * the best-effort low protection. However, we still
3136 : * ideally want to honor how well-behaved groups are in
3137 : * that case instead of simply punishing them all
3138 : * equally. As such, we reclaim them based on how much
3139 : * memory they are using, reducing the scan pressure
3140 : * again by how much of the total memory used is under
3141 : * hard protection.
3142 : */
3143 : unsigned long cgroup_size = mem_cgroup_size(memcg);
3144 : unsigned long protection;
3145 :
3146 : /* memory.low scaling, make sure we retry before OOM */
3147 : if (!sc->memcg_low_reclaim && low > min) {
3148 : protection = low;
3149 : sc->memcg_low_skipped = 1;
3150 : } else {
3151 : protection = min;
3152 : }
3153 :
3154 : /* Avoid TOCTOU with earlier protection check */
3155 : cgroup_size = max(cgroup_size, protection);
3156 :
3157 : scan = lruvec_size - lruvec_size * protection /
3158 : (cgroup_size + 1);
3159 :
3160 : /*
3161 : * Minimally target SWAP_CLUSTER_MAX pages to keep
3162 : * reclaim moving forwards, avoiding decrementing
3163 : * sc->priority further than desirable.
3164 : */
3165 : scan = max(scan, SWAP_CLUSTER_MAX);
3166 : } else {
3167 0 : scan = lruvec_size;
3168 : }
3169 :
3170 0 : scan >>= sc->priority;
3171 :
3172 : /*
3173 : * If the cgroup's already been deleted, make sure to
3174 : * scrape out the remaining cache.
3175 : */
3176 : if (!scan && !mem_cgroup_online(memcg))
3177 : scan = min(lruvec_size, SWAP_CLUSTER_MAX);
3178 :
3179 0 : switch (scan_balance) {
3180 : case SCAN_EQUAL:
3181 : /* Scan lists relative to size */
3182 : break;
3183 : case SCAN_FRACT:
3184 : /*
3185 : * Scan types proportional to swappiness and
3186 : * their relative recent reclaim efficiency.
3187 : * Make sure we don't miss the last page on
3188 : * the offlined memory cgroups because of a
3189 : * round-off error.
3190 : */
3191 0 : scan = mem_cgroup_online(memcg) ?
3192 0 : div64_u64(scan * fraction[file], denominator) :
3193 : DIV64_U64_ROUND_UP(scan * fraction[file],
3194 : denominator);
3195 0 : break;
3196 : case SCAN_FILE:
3197 : case SCAN_ANON:
3198 : /* Scan one type exclusively */
3199 0 : if ((scan_balance == SCAN_FILE) != file)
3200 0 : scan = 0;
3201 : break;
3202 : default:
3203 : /* Look ma, no brain */
3204 0 : BUG();
3205 : }
3206 :
3207 0 : nr[lru] = scan;
3208 : }
3209 0 : }
3210 :
3211 : /*
3212 : * Anonymous LRU management is a waste if there is
3213 : * ultimately no way to reclaim the memory.
3214 : */
3215 : static bool can_age_anon_pages(struct pglist_data *pgdat,
3216 : struct scan_control *sc)
3217 : {
3218 : /* Aging the anon LRU is valuable if swap is present: */
3219 0 : if (total_swap_pages > 0)
3220 : return true;
3221 :
3222 : /* Also valuable if anon pages can be demoted: */
3223 0 : return can_demote(pgdat->node_id, sc);
3224 : }
3225 :
3226 : #ifdef CONFIG_LRU_GEN
3227 :
3228 : #ifdef CONFIG_LRU_GEN_ENABLED
3229 : DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS);
3230 : #define get_cap(cap) static_branch_likely(&lru_gen_caps[cap])
3231 : #else
3232 : DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
3233 : #define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap])
3234 : #endif
3235 :
3236 : /******************************************************************************
3237 : * shorthand helpers
3238 : ******************************************************************************/
3239 :
3240 : #define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset))
3241 :
3242 : #define DEFINE_MAX_SEQ(lruvec) \
3243 : unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
3244 :
3245 : #define DEFINE_MIN_SEQ(lruvec) \
3246 : unsigned long min_seq[ANON_AND_FILE] = { \
3247 : READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
3248 : READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
3249 : }
3250 :
3251 : #define for_each_gen_type_zone(gen, type, zone) \
3252 : for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \
3253 : for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
3254 : for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
3255 :
3256 : #define get_memcg_gen(seq) ((seq) % MEMCG_NR_GENS)
3257 : #define get_memcg_bin(bin) ((bin) % MEMCG_NR_BINS)
3258 :
3259 : static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
3260 : {
3261 : struct pglist_data *pgdat = NODE_DATA(nid);
3262 :
3263 : #ifdef CONFIG_MEMCG
3264 : if (memcg) {
3265 : struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
3266 :
3267 : /* see the comment in mem_cgroup_lruvec() */
3268 : if (!lruvec->pgdat)
3269 : lruvec->pgdat = pgdat;
3270 :
3271 : return lruvec;
3272 : }
3273 : #endif
3274 : VM_WARN_ON_ONCE(!mem_cgroup_disabled());
3275 :
3276 : return &pgdat->__lruvec;
3277 : }
3278 :
3279 : static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
3280 : {
3281 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3282 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
3283 :
3284 : if (!sc->may_swap)
3285 : return 0;
3286 :
3287 : if (!can_demote(pgdat->node_id, sc) &&
3288 : mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
3289 : return 0;
3290 :
3291 : return mem_cgroup_swappiness(memcg);
3292 : }
3293 :
3294 : static int get_nr_gens(struct lruvec *lruvec, int type)
3295 : {
3296 : return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
3297 : }
3298 :
3299 : static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
3300 : {
3301 : /* see the comment on lru_gen_folio */
3302 : return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS &&
3303 : get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) &&
3304 : get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS;
3305 : }
3306 :
3307 : /******************************************************************************
3308 : * Bloom filters
3309 : ******************************************************************************/
3310 :
3311 : /*
3312 : * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when
3313 : * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of
3314 : * bits in a bitmap, k is the number of hash functions and n is the number of
3315 : * inserted items.
3316 : *
3317 : * Page table walkers use one of the two filters to reduce their search space.
3318 : * To get rid of non-leaf entries that no longer have enough leaf entries, the
3319 : * aging uses the double-buffering technique to flip to the other filter each
3320 : * time it produces a new generation. For non-leaf entries that have enough
3321 : * leaf entries, the aging carries them over to the next generation in
3322 : * walk_pmd_range(); the eviction also report them when walking the rmap
3323 : * in lru_gen_look_around().
3324 : *
3325 : * For future optimizations:
3326 : * 1. It's not necessary to keep both filters all the time. The spare one can be
3327 : * freed after the RCU grace period and reallocated if needed again.
3328 : * 2. And when reallocating, it's worth scaling its size according to the number
3329 : * of inserted entries in the other filter, to reduce the memory overhead on
3330 : * small systems and false positives on large systems.
3331 : * 3. Jenkins' hash function is an alternative to Knuth's.
3332 : */
3333 : #define BLOOM_FILTER_SHIFT 15
3334 :
3335 : static inline int filter_gen_from_seq(unsigned long seq)
3336 : {
3337 : return seq % NR_BLOOM_FILTERS;
3338 : }
3339 :
3340 : static void get_item_key(void *item, int *key)
3341 : {
3342 : u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2);
3343 :
3344 : BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32));
3345 :
3346 : key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1);
3347 : key[1] = hash >> BLOOM_FILTER_SHIFT;
3348 : }
3349 :
3350 : static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item)
3351 : {
3352 : int key[2];
3353 : unsigned long *filter;
3354 : int gen = filter_gen_from_seq(seq);
3355 :
3356 : filter = READ_ONCE(lruvec->mm_state.filters[gen]);
3357 : if (!filter)
3358 : return true;
3359 :
3360 : get_item_key(item, key);
3361 :
3362 : return test_bit(key[0], filter) && test_bit(key[1], filter);
3363 : }
3364 :
3365 : static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item)
3366 : {
3367 : int key[2];
3368 : unsigned long *filter;
3369 : int gen = filter_gen_from_seq(seq);
3370 :
3371 : filter = READ_ONCE(lruvec->mm_state.filters[gen]);
3372 : if (!filter)
3373 : return;
3374 :
3375 : get_item_key(item, key);
3376 :
3377 : if (!test_bit(key[0], filter))
3378 : set_bit(key[0], filter);
3379 : if (!test_bit(key[1], filter))
3380 : set_bit(key[1], filter);
3381 : }
3382 :
3383 : static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq)
3384 : {
3385 : unsigned long *filter;
3386 : int gen = filter_gen_from_seq(seq);
3387 :
3388 : filter = lruvec->mm_state.filters[gen];
3389 : if (filter) {
3390 : bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
3391 : return;
3392 : }
3393 :
3394 : filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
3395 : __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
3396 : WRITE_ONCE(lruvec->mm_state.filters[gen], filter);
3397 : }
3398 :
3399 : /******************************************************************************
3400 : * mm_struct list
3401 : ******************************************************************************/
3402 :
3403 : static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg)
3404 : {
3405 : static struct lru_gen_mm_list mm_list = {
3406 : .fifo = LIST_HEAD_INIT(mm_list.fifo),
3407 : .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock),
3408 : };
3409 :
3410 : #ifdef CONFIG_MEMCG
3411 : if (memcg)
3412 : return &memcg->mm_list;
3413 : #endif
3414 : VM_WARN_ON_ONCE(!mem_cgroup_disabled());
3415 :
3416 : return &mm_list;
3417 : }
3418 :
3419 : void lru_gen_add_mm(struct mm_struct *mm)
3420 : {
3421 : int nid;
3422 : struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
3423 : struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3424 :
3425 : VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list));
3426 : #ifdef CONFIG_MEMCG
3427 : VM_WARN_ON_ONCE(mm->lru_gen.memcg);
3428 : mm->lru_gen.memcg = memcg;
3429 : #endif
3430 : spin_lock(&mm_list->lock);
3431 :
3432 : for_each_node_state(nid, N_MEMORY) {
3433 : struct lruvec *lruvec = get_lruvec(memcg, nid);
3434 :
3435 : /* the first addition since the last iteration */
3436 : if (lruvec->mm_state.tail == &mm_list->fifo)
3437 : lruvec->mm_state.tail = &mm->lru_gen.list;
3438 : }
3439 :
3440 : list_add_tail(&mm->lru_gen.list, &mm_list->fifo);
3441 :
3442 : spin_unlock(&mm_list->lock);
3443 : }
3444 :
3445 : void lru_gen_del_mm(struct mm_struct *mm)
3446 : {
3447 : int nid;
3448 : struct lru_gen_mm_list *mm_list;
3449 : struct mem_cgroup *memcg = NULL;
3450 :
3451 : if (list_empty(&mm->lru_gen.list))
3452 : return;
3453 :
3454 : #ifdef CONFIG_MEMCG
3455 : memcg = mm->lru_gen.memcg;
3456 : #endif
3457 : mm_list = get_mm_list(memcg);
3458 :
3459 : spin_lock(&mm_list->lock);
3460 :
3461 : for_each_node(nid) {
3462 : struct lruvec *lruvec = get_lruvec(memcg, nid);
3463 :
3464 : /* where the current iteration continues after */
3465 : if (lruvec->mm_state.head == &mm->lru_gen.list)
3466 : lruvec->mm_state.head = lruvec->mm_state.head->prev;
3467 :
3468 : /* where the last iteration ended before */
3469 : if (lruvec->mm_state.tail == &mm->lru_gen.list)
3470 : lruvec->mm_state.tail = lruvec->mm_state.tail->next;
3471 : }
3472 :
3473 : list_del_init(&mm->lru_gen.list);
3474 :
3475 : spin_unlock(&mm_list->lock);
3476 :
3477 : #ifdef CONFIG_MEMCG
3478 : mem_cgroup_put(mm->lru_gen.memcg);
3479 : mm->lru_gen.memcg = NULL;
3480 : #endif
3481 : }
3482 :
3483 : #ifdef CONFIG_MEMCG
3484 : void lru_gen_migrate_mm(struct mm_struct *mm)
3485 : {
3486 : struct mem_cgroup *memcg;
3487 : struct task_struct *task = rcu_dereference_protected(mm->owner, true);
3488 :
3489 : VM_WARN_ON_ONCE(task->mm != mm);
3490 : lockdep_assert_held(&task->alloc_lock);
3491 :
3492 : /* for mm_update_next_owner() */
3493 : if (mem_cgroup_disabled())
3494 : return;
3495 :
3496 : /* migration can happen before addition */
3497 : if (!mm->lru_gen.memcg)
3498 : return;
3499 :
3500 : rcu_read_lock();
3501 : memcg = mem_cgroup_from_task(task);
3502 : rcu_read_unlock();
3503 : if (memcg == mm->lru_gen.memcg)
3504 : return;
3505 :
3506 : VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list));
3507 :
3508 : lru_gen_del_mm(mm);
3509 : lru_gen_add_mm(mm);
3510 : }
3511 : #endif
3512 :
3513 : static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last)
3514 : {
3515 : int i;
3516 : int hist;
3517 :
3518 : lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock);
3519 :
3520 : if (walk) {
3521 : hist = lru_hist_from_seq(walk->max_seq);
3522 :
3523 : for (i = 0; i < NR_MM_STATS; i++) {
3524 : WRITE_ONCE(lruvec->mm_state.stats[hist][i],
3525 : lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]);
3526 : walk->mm_stats[i] = 0;
3527 : }
3528 : }
3529 :
3530 : if (NR_HIST_GENS > 1 && last) {
3531 : hist = lru_hist_from_seq(lruvec->mm_state.seq + 1);
3532 :
3533 : for (i = 0; i < NR_MM_STATS; i++)
3534 : WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0);
3535 : }
3536 : }
3537 :
3538 : static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
3539 : {
3540 : int type;
3541 : unsigned long size = 0;
3542 : struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
3543 : int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);
3544 :
3545 : if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap))
3546 : return true;
3547 :
3548 : clear_bit(key, &mm->lru_gen.bitmap);
3549 :
3550 : for (type = !walk->can_swap; type < ANON_AND_FILE; type++) {
3551 : size += type ? get_mm_counter(mm, MM_FILEPAGES) :
3552 : get_mm_counter(mm, MM_ANONPAGES) +
3553 : get_mm_counter(mm, MM_SHMEMPAGES);
3554 : }
3555 :
3556 : if (size < MIN_LRU_BATCH)
3557 : return true;
3558 :
3559 : return !mmget_not_zero(mm);
3560 : }
3561 :
3562 : static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk,
3563 : struct mm_struct **iter)
3564 : {
3565 : bool first = false;
3566 : bool last = false;
3567 : struct mm_struct *mm = NULL;
3568 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3569 : struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3570 : struct lru_gen_mm_state *mm_state = &lruvec->mm_state;
3571 :
3572 : /*
3573 : * mm_state->seq is incremented after each iteration of mm_list. There
3574 : * are three interesting cases for this page table walker:
3575 : * 1. It tries to start a new iteration with a stale max_seq: there is
3576 : * nothing left to do.
3577 : * 2. It started the next iteration: it needs to reset the Bloom filter
3578 : * so that a fresh set of PTE tables can be recorded.
3579 : * 3. It ended the current iteration: it needs to reset the mm stats
3580 : * counters and tell its caller to increment max_seq.
3581 : */
3582 : spin_lock(&mm_list->lock);
3583 :
3584 : VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq);
3585 :
3586 : if (walk->max_seq <= mm_state->seq)
3587 : goto done;
3588 :
3589 : if (!mm_state->head)
3590 : mm_state->head = &mm_list->fifo;
3591 :
3592 : if (mm_state->head == &mm_list->fifo)
3593 : first = true;
3594 :
3595 : do {
3596 : mm_state->head = mm_state->head->next;
3597 : if (mm_state->head == &mm_list->fifo) {
3598 : WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
3599 : last = true;
3600 : break;
3601 : }
3602 :
3603 : /* force scan for those added after the last iteration */
3604 : if (!mm_state->tail || mm_state->tail == mm_state->head) {
3605 : mm_state->tail = mm_state->head->next;
3606 : walk->force_scan = true;
3607 : }
3608 :
3609 : mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
3610 : if (should_skip_mm(mm, walk))
3611 : mm = NULL;
3612 : } while (!mm);
3613 : done:
3614 : if (*iter || last)
3615 : reset_mm_stats(lruvec, walk, last);
3616 :
3617 : spin_unlock(&mm_list->lock);
3618 :
3619 : if (mm && first)
3620 : reset_bloom_filter(lruvec, walk->max_seq + 1);
3621 :
3622 : if (*iter)
3623 : mmput_async(*iter);
3624 :
3625 : *iter = mm;
3626 :
3627 : return last;
3628 : }
3629 :
3630 : static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)
3631 : {
3632 : bool success = false;
3633 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
3634 : struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
3635 : struct lru_gen_mm_state *mm_state = &lruvec->mm_state;
3636 :
3637 : spin_lock(&mm_list->lock);
3638 :
3639 : VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
3640 :
3641 : if (max_seq > mm_state->seq) {
3642 : mm_state->head = NULL;
3643 : mm_state->tail = NULL;
3644 : WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
3645 : reset_mm_stats(lruvec, NULL, true);
3646 : success = true;
3647 : }
3648 :
3649 : spin_unlock(&mm_list->lock);
3650 :
3651 : return success;
3652 : }
3653 :
3654 : /******************************************************************************
3655 : * PID controller
3656 : ******************************************************************************/
3657 :
3658 : /*
3659 : * A feedback loop based on Proportional-Integral-Derivative (PID) controller.
3660 : *
3661 : * The P term is refaulted/(evicted+protected) from a tier in the generation
3662 : * currently being evicted; the I term is the exponential moving average of the
3663 : * P term over the generations previously evicted, using the smoothing factor
3664 : * 1/2; the D term isn't supported.
3665 : *
3666 : * The setpoint (SP) is always the first tier of one type; the process variable
3667 : * (PV) is either any tier of the other type or any other tier of the same
3668 : * type.
3669 : *
3670 : * The error is the difference between the SP and the PV; the correction is to
3671 : * turn off protection when SP>PV or turn on protection when SP<PV.
3672 : *
3673 : * For future optimizations:
3674 : * 1. The D term may discount the other two terms over time so that long-lived
3675 : * generations can resist stale information.
3676 : */
3677 : struct ctrl_pos {
3678 : unsigned long refaulted;
3679 : unsigned long total;
3680 : int gain;
3681 : };
3682 :
3683 : static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
3684 : struct ctrl_pos *pos)
3685 : {
3686 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
3687 : int hist = lru_hist_from_seq(lrugen->min_seq[type]);
3688 :
3689 : pos->refaulted = lrugen->avg_refaulted[type][tier] +
3690 : atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3691 : pos->total = lrugen->avg_total[type][tier] +
3692 : atomic_long_read(&lrugen->evicted[hist][type][tier]);
3693 : if (tier)
3694 : pos->total += lrugen->protected[hist][type][tier - 1];
3695 : pos->gain = gain;
3696 : }
3697 :
3698 : static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
3699 : {
3700 : int hist, tier;
3701 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
3702 : bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
3703 : unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
3704 :
3705 : lockdep_assert_held(&lruvec->lru_lock);
3706 :
3707 : if (!carryover && !clear)
3708 : return;
3709 :
3710 : hist = lru_hist_from_seq(seq);
3711 :
3712 : for (tier = 0; tier < MAX_NR_TIERS; tier++) {
3713 : if (carryover) {
3714 : unsigned long sum;
3715 :
3716 : sum = lrugen->avg_refaulted[type][tier] +
3717 : atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3718 : WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);
3719 :
3720 : sum = lrugen->avg_total[type][tier] +
3721 : atomic_long_read(&lrugen->evicted[hist][type][tier]);
3722 : if (tier)
3723 : sum += lrugen->protected[hist][type][tier - 1];
3724 : WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);
3725 : }
3726 :
3727 : if (clear) {
3728 : atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);
3729 : atomic_long_set(&lrugen->evicted[hist][type][tier], 0);
3730 : if (tier)
3731 : WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0);
3732 : }
3733 : }
3734 : }
3735 :
3736 : static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
3737 : {
3738 : /*
3739 : * Return true if the PV has a limited number of refaults or a lower
3740 : * refaulted/total than the SP.
3741 : */
3742 : return pv->refaulted < MIN_LRU_BATCH ||
3743 : pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <=
3744 : (sp->refaulted + 1) * pv->total * pv->gain;
3745 : }
3746 :
3747 : /******************************************************************************
3748 : * the aging
3749 : ******************************************************************************/
3750 :
3751 : /* promote pages accessed through page tables */
3752 : static int folio_update_gen(struct folio *folio, int gen)
3753 : {
3754 : unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
3755 :
3756 : VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
3757 : VM_WARN_ON_ONCE(!rcu_read_lock_held());
3758 :
3759 : do {
3760 : /* lru_gen_del_folio() has isolated this page? */
3761 : if (!(old_flags & LRU_GEN_MASK)) {
3762 : /* for shrink_folio_list() */
3763 : new_flags = old_flags | BIT(PG_referenced);
3764 : continue;
3765 : }
3766 :
3767 : new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
3768 : new_flags |= (gen + 1UL) << LRU_GEN_PGOFF;
3769 : } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
3770 :
3771 : return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
3772 : }
3773 :
3774 : /* protect pages accessed multiple times through file descriptors */
3775 : static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
3776 : {
3777 : int type = folio_is_file_lru(folio);
3778 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
3779 : int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3780 : unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
3781 :
3782 : VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
3783 :
3784 : do {
3785 : new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
3786 : /* folio_update_gen() has promoted this page? */
3787 : if (new_gen >= 0 && new_gen != old_gen)
3788 : return new_gen;
3789 :
3790 : new_gen = (old_gen + 1) % MAX_NR_GENS;
3791 :
3792 : new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
3793 : new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
3794 : /* for folio_end_writeback() */
3795 : if (reclaiming)
3796 : new_flags |= BIT(PG_reclaim);
3797 : } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
3798 :
3799 : lru_gen_update_size(lruvec, folio, old_gen, new_gen);
3800 :
3801 : return new_gen;
3802 : }
3803 :
3804 : static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
3805 : int old_gen, int new_gen)
3806 : {
3807 : int type = folio_is_file_lru(folio);
3808 : int zone = folio_zonenum(folio);
3809 : int delta = folio_nr_pages(folio);
3810 :
3811 : VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS);
3812 : VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS);
3813 :
3814 : walk->batched++;
3815 :
3816 : walk->nr_pages[old_gen][type][zone] -= delta;
3817 : walk->nr_pages[new_gen][type][zone] += delta;
3818 : }
3819 :
3820 : static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk)
3821 : {
3822 : int gen, type, zone;
3823 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
3824 :
3825 : walk->batched = 0;
3826 :
3827 : for_each_gen_type_zone(gen, type, zone) {
3828 : enum lru_list lru = type * LRU_INACTIVE_FILE;
3829 : int delta = walk->nr_pages[gen][type][zone];
3830 :
3831 : if (!delta)
3832 : continue;
3833 :
3834 : walk->nr_pages[gen][type][zone] = 0;
3835 : WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
3836 : lrugen->nr_pages[gen][type][zone] + delta);
3837 :
3838 : if (lru_gen_is_active(lruvec, gen))
3839 : lru += LRU_ACTIVE;
3840 : __update_lru_size(lruvec, lru, zone, delta);
3841 : }
3842 : }
3843 :
3844 : static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args)
3845 : {
3846 : struct address_space *mapping;
3847 : struct vm_area_struct *vma = args->vma;
3848 : struct lru_gen_mm_walk *walk = args->private;
3849 :
3850 : if (!vma_is_accessible(vma))
3851 : return true;
3852 :
3853 : if (is_vm_hugetlb_page(vma))
3854 : return true;
3855 :
3856 : if (!vma_has_recency(vma))
3857 : return true;
3858 :
3859 : if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL))
3860 : return true;
3861 :
3862 : if (vma == get_gate_vma(vma->vm_mm))
3863 : return true;
3864 :
3865 : if (vma_is_anonymous(vma))
3866 : return !walk->can_swap;
3867 :
3868 : if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping))
3869 : return true;
3870 :
3871 : mapping = vma->vm_file->f_mapping;
3872 : if (mapping_unevictable(mapping))
3873 : return true;
3874 :
3875 : if (shmem_mapping(mapping))
3876 : return !walk->can_swap;
3877 :
3878 : /* to exclude special mappings like dax, etc. */
3879 : return !mapping->a_ops->read_folio;
3880 : }
3881 :
3882 : /*
3883 : * Some userspace memory allocators map many single-page VMAs. Instead of
3884 : * returning back to the PGD table for each of such VMAs, finish an entire PMD
3885 : * table to reduce zigzags and improve cache performance.
3886 : */
3887 : static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args,
3888 : unsigned long *vm_start, unsigned long *vm_end)
3889 : {
3890 : unsigned long start = round_up(*vm_end, size);
3891 : unsigned long end = (start | ~mask) + 1;
3892 : VMA_ITERATOR(vmi, args->mm, start);
3893 :
3894 : VM_WARN_ON_ONCE(mask & size);
3895 : VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask));
3896 :
3897 : for_each_vma(vmi, args->vma) {
3898 : if (end && end <= args->vma->vm_start)
3899 : return false;
3900 :
3901 : if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args))
3902 : continue;
3903 :
3904 : *vm_start = max(start, args->vma->vm_start);
3905 : *vm_end = min(end - 1, args->vma->vm_end - 1) + 1;
3906 :
3907 : return true;
3908 : }
3909 :
3910 : return false;
3911 : }
3912 :
3913 : static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr)
3914 : {
3915 : unsigned long pfn = pte_pfn(pte);
3916 :
3917 : VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
3918 :
3919 : if (!pte_present(pte) || is_zero_pfn(pfn))
3920 : return -1;
3921 :
3922 : if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte)))
3923 : return -1;
3924 :
3925 : if (WARN_ON_ONCE(!pfn_valid(pfn)))
3926 : return -1;
3927 :
3928 : return pfn;
3929 : }
3930 :
3931 : #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
3932 : static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr)
3933 : {
3934 : unsigned long pfn = pmd_pfn(pmd);
3935 :
3936 : VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
3937 :
3938 : if (!pmd_present(pmd) || is_huge_zero_pmd(pmd))
3939 : return -1;
3940 :
3941 : if (WARN_ON_ONCE(pmd_devmap(pmd)))
3942 : return -1;
3943 :
3944 : if (WARN_ON_ONCE(!pfn_valid(pfn)))
3945 : return -1;
3946 :
3947 : return pfn;
3948 : }
3949 : #endif
3950 :
3951 : static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg,
3952 : struct pglist_data *pgdat, bool can_swap)
3953 : {
3954 : struct folio *folio;
3955 :
3956 : /* try to avoid unnecessary memory loads */
3957 : if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
3958 : return NULL;
3959 :
3960 : folio = pfn_folio(pfn);
3961 : if (folio_nid(folio) != pgdat->node_id)
3962 : return NULL;
3963 :
3964 : if (folio_memcg_rcu(folio) != memcg)
3965 : return NULL;
3966 :
3967 : /* file VMAs can contain anon pages from COW */
3968 : if (!folio_is_file_lru(folio) && !can_swap)
3969 : return NULL;
3970 :
3971 : return folio;
3972 : }
3973 :
3974 : static bool suitable_to_scan(int total, int young)
3975 : {
3976 : int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8);
3977 :
3978 : /* suitable if the average number of young PTEs per cacheline is >=1 */
3979 : return young * n >= total;
3980 : }
3981 :
3982 : static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
3983 : struct mm_walk *args)
3984 : {
3985 : int i;
3986 : pte_t *pte;
3987 : spinlock_t *ptl;
3988 : unsigned long addr;
3989 : int total = 0;
3990 : int young = 0;
3991 : struct lru_gen_mm_walk *walk = args->private;
3992 : struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
3993 : struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
3994 : int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
3995 :
3996 : VM_WARN_ON_ONCE(pmd_leaf(*pmd));
3997 :
3998 : ptl = pte_lockptr(args->mm, pmd);
3999 : if (!spin_trylock(ptl))
4000 : return false;
4001 :
4002 : arch_enter_lazy_mmu_mode();
4003 :
4004 : pte = pte_offset_map(pmd, start & PMD_MASK);
4005 : restart:
4006 : for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) {
4007 : unsigned long pfn;
4008 : struct folio *folio;
4009 :
4010 : total++;
4011 : walk->mm_stats[MM_LEAF_TOTAL]++;
4012 :
4013 : pfn = get_pte_pfn(pte[i], args->vma, addr);
4014 : if (pfn == -1)
4015 : continue;
4016 :
4017 : if (!pte_young(pte[i])) {
4018 : walk->mm_stats[MM_LEAF_OLD]++;
4019 : continue;
4020 : }
4021 :
4022 : folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);
4023 : if (!folio)
4024 : continue;
4025 :
4026 : if (!ptep_test_and_clear_young(args->vma, addr, pte + i))
4027 : VM_WARN_ON_ONCE(true);
4028 :
4029 : young++;
4030 : walk->mm_stats[MM_LEAF_YOUNG]++;
4031 :
4032 : if (pte_dirty(pte[i]) && !folio_test_dirty(folio) &&
4033 : !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
4034 : !folio_test_swapcache(folio)))
4035 : folio_mark_dirty(folio);
4036 :
4037 : old_gen = folio_update_gen(folio, new_gen);
4038 : if (old_gen >= 0 && old_gen != new_gen)
4039 : update_batch_size(walk, folio, old_gen, new_gen);
4040 : }
4041 :
4042 : if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
4043 : goto restart;
4044 :
4045 : pte_unmap(pte);
4046 :
4047 : arch_leave_lazy_mmu_mode();
4048 : spin_unlock(ptl);
4049 :
4050 : return suitable_to_scan(total, young);
4051 : }
4052 :
4053 : #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
4054 : static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
4055 : struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
4056 : {
4057 : int i;
4058 : pmd_t *pmd;
4059 : spinlock_t *ptl;
4060 : struct lru_gen_mm_walk *walk = args->private;
4061 : struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
4062 : struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
4063 : int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
4064 :
4065 : VM_WARN_ON_ONCE(pud_leaf(*pud));
4066 :
4067 : /* try to batch at most 1+MIN_LRU_BATCH+1 entries */
4068 : if (*first == -1) {
4069 : *first = addr;
4070 : bitmap_zero(bitmap, MIN_LRU_BATCH);
4071 : return;
4072 : }
4073 :
4074 : i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
4075 : if (i && i <= MIN_LRU_BATCH) {
4076 : __set_bit(i - 1, bitmap);
4077 : return;
4078 : }
4079 :
4080 : pmd = pmd_offset(pud, *first);
4081 :
4082 : ptl = pmd_lockptr(args->mm, pmd);
4083 : if (!spin_trylock(ptl))
4084 : goto done;
4085 :
4086 : arch_enter_lazy_mmu_mode();
4087 :
4088 : do {
4089 : unsigned long pfn;
4090 : struct folio *folio;
4091 :
4092 : /* don't round down the first address */
4093 : addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
4094 :
4095 : pfn = get_pmd_pfn(pmd[i], vma, addr);
4096 : if (pfn == -1)
4097 : goto next;
4098 :
4099 : if (!pmd_trans_huge(pmd[i])) {
4100 : if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
4101 : pmdp_test_and_clear_young(vma, addr, pmd + i);
4102 : goto next;
4103 : }
4104 :
4105 : folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);
4106 : if (!folio)
4107 : goto next;
4108 :
4109 : if (!pmdp_test_and_clear_young(vma, addr, pmd + i))
4110 : goto next;
4111 :
4112 : walk->mm_stats[MM_LEAF_YOUNG]++;
4113 :
4114 : if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) &&
4115 : !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
4116 : !folio_test_swapcache(folio)))
4117 : folio_mark_dirty(folio);
4118 :
4119 : old_gen = folio_update_gen(folio, new_gen);
4120 : if (old_gen >= 0 && old_gen != new_gen)
4121 : update_batch_size(walk, folio, old_gen, new_gen);
4122 : next:
4123 : i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
4124 : } while (i <= MIN_LRU_BATCH);
4125 :
4126 : arch_leave_lazy_mmu_mode();
4127 : spin_unlock(ptl);
4128 : done:
4129 : *first = -1;
4130 : }
4131 : #else
4132 : static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
4133 : struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
4134 : {
4135 : }
4136 : #endif
4137 :
4138 : static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
4139 : struct mm_walk *args)
4140 : {
4141 : int i;
4142 : pmd_t *pmd;
4143 : unsigned long next;
4144 : unsigned long addr;
4145 : struct vm_area_struct *vma;
4146 : unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];
4147 : unsigned long first = -1;
4148 : struct lru_gen_mm_walk *walk = args->private;
4149 :
4150 : VM_WARN_ON_ONCE(pud_leaf(*pud));
4151 :
4152 : /*
4153 : * Finish an entire PMD in two passes: the first only reaches to PTE
4154 : * tables to avoid taking the PMD lock; the second, if necessary, takes
4155 : * the PMD lock to clear the accessed bit in PMD entries.
4156 : */
4157 : pmd = pmd_offset(pud, start & PUD_MASK);
4158 : restart:
4159 : /* walk_pte_range() may call get_next_vma() */
4160 : vma = args->vma;
4161 : for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) {
4162 : pmd_t val = pmdp_get_lockless(pmd + i);
4163 :
4164 : next = pmd_addr_end(addr, end);
4165 :
4166 : if (!pmd_present(val) || is_huge_zero_pmd(val)) {
4167 : walk->mm_stats[MM_LEAF_TOTAL]++;
4168 : continue;
4169 : }
4170 :
4171 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4172 : if (pmd_trans_huge(val)) {
4173 : unsigned long pfn = pmd_pfn(val);
4174 : struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
4175 :
4176 : walk->mm_stats[MM_LEAF_TOTAL]++;
4177 :
4178 : if (!pmd_young(val)) {
4179 : walk->mm_stats[MM_LEAF_OLD]++;
4180 : continue;
4181 : }
4182 :
4183 : /* try to avoid unnecessary memory loads */
4184 : if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
4185 : continue;
4186 :
4187 : walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
4188 : continue;
4189 : }
4190 : #endif
4191 : walk->mm_stats[MM_NONLEAF_TOTAL]++;
4192 :
4193 : if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {
4194 : if (!pmd_young(val))
4195 : continue;
4196 :
4197 : walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
4198 : }
4199 :
4200 : if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
4201 : continue;
4202 :
4203 : walk->mm_stats[MM_NONLEAF_FOUND]++;
4204 :
4205 : if (!walk_pte_range(&val, addr, next, args))
4206 : continue;
4207 :
4208 : walk->mm_stats[MM_NONLEAF_ADDED]++;
4209 :
4210 : /* carry over to the next generation */
4211 : update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
4212 : }
4213 :
4214 : walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
4215 :
4216 : if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
4217 : goto restart;
4218 : }
4219 :
4220 : static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
4221 : struct mm_walk *args)
4222 : {
4223 : int i;
4224 : pud_t *pud;
4225 : unsigned long addr;
4226 : unsigned long next;
4227 : struct lru_gen_mm_walk *walk = args->private;
4228 :
4229 : VM_WARN_ON_ONCE(p4d_leaf(*p4d));
4230 :
4231 : pud = pud_offset(p4d, start & P4D_MASK);
4232 : restart:
4233 : for (i = pud_index(start), addr = start; addr != end; i++, addr = next) {
4234 : pud_t val = READ_ONCE(pud[i]);
4235 :
4236 : next = pud_addr_end(addr, end);
4237 :
4238 : if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
4239 : continue;
4240 :
4241 : walk_pmd_range(&val, addr, next, args);
4242 :
4243 : if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
4244 : end = (addr | ~PUD_MASK) + 1;
4245 : goto done;
4246 : }
4247 : }
4248 :
4249 : if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end))
4250 : goto restart;
4251 :
4252 : end = round_up(end, P4D_SIZE);
4253 : done:
4254 : if (!end || !args->vma)
4255 : return 1;
4256 :
4257 : walk->next_addr = max(end, args->vma->vm_start);
4258 :
4259 : return -EAGAIN;
4260 : }
4261 :
4262 : static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk)
4263 : {
4264 : static const struct mm_walk_ops mm_walk_ops = {
4265 : .test_walk = should_skip_vma,
4266 : .p4d_entry = walk_pud_range,
4267 : };
4268 :
4269 : int err;
4270 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4271 :
4272 : walk->next_addr = FIRST_USER_ADDRESS;
4273 :
4274 : do {
4275 : DEFINE_MAX_SEQ(lruvec);
4276 :
4277 : err = -EBUSY;
4278 :
4279 : /* another thread might have called inc_max_seq() */
4280 : if (walk->max_seq != max_seq)
4281 : break;
4282 :
4283 : /* folio_update_gen() requires stable folio_memcg() */
4284 : if (!mem_cgroup_trylock_pages(memcg))
4285 : break;
4286 :
4287 : /* the caller might be holding the lock for write */
4288 : if (mmap_read_trylock(mm)) {
4289 : err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk);
4290 :
4291 : mmap_read_unlock(mm);
4292 : }
4293 :
4294 : mem_cgroup_unlock_pages();
4295 :
4296 : if (walk->batched) {
4297 : spin_lock_irq(&lruvec->lru_lock);
4298 : reset_batch_size(lruvec, walk);
4299 : spin_unlock_irq(&lruvec->lru_lock);
4300 : }
4301 :
4302 : cond_resched();
4303 : } while (err == -EAGAIN);
4304 : }
4305 :
4306 : static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc)
4307 : {
4308 : struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
4309 :
4310 : if (pgdat && current_is_kswapd()) {
4311 : VM_WARN_ON_ONCE(walk);
4312 :
4313 : walk = &pgdat->mm_walk;
4314 : } else if (!walk && force_alloc) {
4315 : VM_WARN_ON_ONCE(current_is_kswapd());
4316 :
4317 : walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
4318 : }
4319 :
4320 : current->reclaim_state->mm_walk = walk;
4321 :
4322 : return walk;
4323 : }
4324 :
4325 : static void clear_mm_walk(void)
4326 : {
4327 : struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
4328 :
4329 : VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages)));
4330 : VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats)));
4331 :
4332 : current->reclaim_state->mm_walk = NULL;
4333 :
4334 : if (!current_is_kswapd())
4335 : kfree(walk);
4336 : }
4337 :
4338 : static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
4339 : {
4340 : int zone;
4341 : int remaining = MAX_LRU_BATCH;
4342 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
4343 : int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
4344 :
4345 : if (type == LRU_GEN_ANON && !can_swap)
4346 : goto done;
4347 :
4348 : /* prevent cold/hot inversion if force_scan is true */
4349 : for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4350 : struct list_head *head = &lrugen->folios[old_gen][type][zone];
4351 :
4352 : while (!list_empty(head)) {
4353 : struct folio *folio = lru_to_folio(head);
4354 :
4355 : VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
4356 : VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
4357 : VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
4358 : VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
4359 :
4360 : new_gen = folio_inc_gen(lruvec, folio, false);
4361 : list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
4362 :
4363 : if (!--remaining)
4364 : return false;
4365 : }
4366 : }
4367 : done:
4368 : reset_ctrl_pos(lruvec, type, true);
4369 : WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);
4370 :
4371 : return true;
4372 : }
4373 :
4374 : static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
4375 : {
4376 : int gen, type, zone;
4377 : bool success = false;
4378 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
4379 : DEFINE_MIN_SEQ(lruvec);
4380 :
4381 : VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
4382 :
4383 : /* find the oldest populated generation */
4384 : for (type = !can_swap; type < ANON_AND_FILE; type++) {
4385 : while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
4386 : gen = lru_gen_from_seq(min_seq[type]);
4387 :
4388 : for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4389 : if (!list_empty(&lrugen->folios[gen][type][zone]))
4390 : goto next;
4391 : }
4392 :
4393 : min_seq[type]++;
4394 : }
4395 : next:
4396 : ;
4397 : }
4398 :
4399 : /* see the comment on lru_gen_folio */
4400 : if (can_swap) {
4401 : min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]);
4402 : min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
4403 : }
4404 :
4405 : for (type = !can_swap; type < ANON_AND_FILE; type++) {
4406 : if (min_seq[type] == lrugen->min_seq[type])
4407 : continue;
4408 :
4409 : reset_ctrl_pos(lruvec, type, true);
4410 : WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
4411 : success = true;
4412 : }
4413 :
4414 : return success;
4415 : }
4416 :
4417 : static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)
4418 : {
4419 : int prev, next;
4420 : int type, zone;
4421 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
4422 :
4423 : spin_lock_irq(&lruvec->lru_lock);
4424 :
4425 : VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
4426 :
4427 : for (type = ANON_AND_FILE - 1; type >= 0; type--) {
4428 : if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
4429 : continue;
4430 :
4431 : VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap));
4432 :
4433 : while (!inc_min_seq(lruvec, type, can_swap)) {
4434 : spin_unlock_irq(&lruvec->lru_lock);
4435 : cond_resched();
4436 : spin_lock_irq(&lruvec->lru_lock);
4437 : }
4438 : }
4439 :
4440 : /*
4441 : * Update the active/inactive LRU sizes for compatibility. Both sides of
4442 : * the current max_seq need to be covered, since max_seq+1 can overlap
4443 : * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do
4444 : * overlap, cold/hot inversion happens.
4445 : */
4446 : prev = lru_gen_from_seq(lrugen->max_seq - 1);
4447 : next = lru_gen_from_seq(lrugen->max_seq + 1);
4448 :
4449 : for (type = 0; type < ANON_AND_FILE; type++) {
4450 : for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4451 : enum lru_list lru = type * LRU_INACTIVE_FILE;
4452 : long delta = lrugen->nr_pages[prev][type][zone] -
4453 : lrugen->nr_pages[next][type][zone];
4454 :
4455 : if (!delta)
4456 : continue;
4457 :
4458 : __update_lru_size(lruvec, lru, zone, delta);
4459 : __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta);
4460 : }
4461 : }
4462 :
4463 : for (type = 0; type < ANON_AND_FILE; type++)
4464 : reset_ctrl_pos(lruvec, type, false);
4465 :
4466 : WRITE_ONCE(lrugen->timestamps[next], jiffies);
4467 : /* make sure preceding modifications appear */
4468 : smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
4469 :
4470 : spin_unlock_irq(&lruvec->lru_lock);
4471 : }
4472 :
4473 : static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
4474 : struct scan_control *sc, bool can_swap, bool force_scan)
4475 : {
4476 : bool success;
4477 : struct lru_gen_mm_walk *walk;
4478 : struct mm_struct *mm = NULL;
4479 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
4480 :
4481 : VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
4482 :
4483 : /* see the comment in iterate_mm_list() */
4484 : if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) {
4485 : success = false;
4486 : goto done;
4487 : }
4488 :
4489 : /*
4490 : * If the hardware doesn't automatically set the accessed bit, fallback
4491 : * to lru_gen_look_around(), which only clears the accessed bit in a
4492 : * handful of PTEs. Spreading the work out over a period of time usually
4493 : * is less efficient, but it avoids bursty page faults.
4494 : */
4495 : if (!arch_has_hw_pte_young() || !get_cap(LRU_GEN_MM_WALK)) {
4496 : success = iterate_mm_list_nowalk(lruvec, max_seq);
4497 : goto done;
4498 : }
4499 :
4500 : walk = set_mm_walk(NULL, true);
4501 : if (!walk) {
4502 : success = iterate_mm_list_nowalk(lruvec, max_seq);
4503 : goto done;
4504 : }
4505 :
4506 : walk->lruvec = lruvec;
4507 : walk->max_seq = max_seq;
4508 : walk->can_swap = can_swap;
4509 : walk->force_scan = force_scan;
4510 :
4511 : do {
4512 : success = iterate_mm_list(lruvec, walk, &mm);
4513 : if (mm)
4514 : walk_mm(lruvec, mm, walk);
4515 : } while (mm);
4516 : done:
4517 : if (success)
4518 : inc_max_seq(lruvec, can_swap, force_scan);
4519 :
4520 : return success;
4521 : }
4522 :
4523 : /******************************************************************************
4524 : * working set protection
4525 : ******************************************************************************/
4526 :
4527 : static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
4528 : {
4529 : int gen, type, zone;
4530 : unsigned long total = 0;
4531 : bool can_swap = get_swappiness(lruvec, sc);
4532 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
4533 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4534 : DEFINE_MAX_SEQ(lruvec);
4535 : DEFINE_MIN_SEQ(lruvec);
4536 :
4537 : for (type = !can_swap; type < ANON_AND_FILE; type++) {
4538 : unsigned long seq;
4539 :
4540 : for (seq = min_seq[type]; seq <= max_seq; seq++) {
4541 : gen = lru_gen_from_seq(seq);
4542 :
4543 : for (zone = 0; zone < MAX_NR_ZONES; zone++)
4544 : total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
4545 : }
4546 : }
4547 :
4548 : /* whether the size is big enough to be helpful */
4549 : return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
4550 : }
4551 :
4552 : static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc,
4553 : unsigned long min_ttl)
4554 : {
4555 : int gen;
4556 : unsigned long birth;
4557 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4558 : DEFINE_MIN_SEQ(lruvec);
4559 :
4560 : /* see the comment on lru_gen_folio */
4561 : gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]);
4562 : birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
4563 :
4564 : if (time_is_after_jiffies(birth + min_ttl))
4565 : return false;
4566 :
4567 : if (!lruvec_is_sizable(lruvec, sc))
4568 : return false;
4569 :
4570 : mem_cgroup_calculate_protection(NULL, memcg);
4571 :
4572 : return !mem_cgroup_below_min(NULL, memcg);
4573 : }
4574 :
4575 : /* to protect the working set of the last N jiffies */
4576 : static unsigned long lru_gen_min_ttl __read_mostly;
4577 :
4578 : static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
4579 : {
4580 : struct mem_cgroup *memcg;
4581 : unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl);
4582 :
4583 : VM_WARN_ON_ONCE(!current_is_kswapd());
4584 :
4585 : /* check the order to exclude compaction-induced reclaim */
4586 : if (!min_ttl || sc->order || sc->priority == DEF_PRIORITY)
4587 : return;
4588 :
4589 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
4590 : do {
4591 : struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
4592 :
4593 : if (lruvec_is_reclaimable(lruvec, sc, min_ttl)) {
4594 : mem_cgroup_iter_break(NULL, memcg);
4595 : return;
4596 : }
4597 :
4598 : cond_resched();
4599 : } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
4600 :
4601 : /*
4602 : * The main goal is to OOM kill if every generation from all memcgs is
4603 : * younger than min_ttl. However, another possibility is all memcgs are
4604 : * either too small or below min.
4605 : */
4606 : if (mutex_trylock(&oom_lock)) {
4607 : struct oom_control oc = {
4608 : .gfp_mask = sc->gfp_mask,
4609 : };
4610 :
4611 : out_of_memory(&oc);
4612 :
4613 : mutex_unlock(&oom_lock);
4614 : }
4615 : }
4616 :
4617 : /******************************************************************************
4618 : * rmap/PT walk feedback
4619 : ******************************************************************************/
4620 :
4621 : /*
4622 : * This function exploits spatial locality when shrink_folio_list() walks the
4623 : * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
4624 : * the scan was done cacheline efficiently, it adds the PMD entry pointing to
4625 : * the PTE table to the Bloom filter. This forms a feedback loop between the
4626 : * eviction and the aging.
4627 : */
4628 : void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
4629 : {
4630 : int i;
4631 : unsigned long start;
4632 : unsigned long end;
4633 : struct lru_gen_mm_walk *walk;
4634 : int young = 0;
4635 : pte_t *pte = pvmw->pte;
4636 : unsigned long addr = pvmw->address;
4637 : struct folio *folio = pfn_folio(pvmw->pfn);
4638 : struct mem_cgroup *memcg = folio_memcg(folio);
4639 : struct pglist_data *pgdat = folio_pgdat(folio);
4640 : struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
4641 : DEFINE_MAX_SEQ(lruvec);
4642 : int old_gen, new_gen = lru_gen_from_seq(max_seq);
4643 :
4644 : lockdep_assert_held(pvmw->ptl);
4645 : VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
4646 :
4647 : if (spin_is_contended(pvmw->ptl))
4648 : return;
4649 :
4650 : /* avoid taking the LRU lock under the PTL when possible */
4651 : walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL;
4652 :
4653 : start = max(addr & PMD_MASK, pvmw->vma->vm_start);
4654 : end = min(addr | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1;
4655 :
4656 : if (end - start > MIN_LRU_BATCH * PAGE_SIZE) {
4657 : if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2)
4658 : end = start + MIN_LRU_BATCH * PAGE_SIZE;
4659 : else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2)
4660 : start = end - MIN_LRU_BATCH * PAGE_SIZE;
4661 : else {
4662 : start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2;
4663 : end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2;
4664 : }
4665 : }
4666 :
4667 : /* folio_update_gen() requires stable folio_memcg() */
4668 : if (!mem_cgroup_trylock_pages(memcg))
4669 : return;
4670 :
4671 : arch_enter_lazy_mmu_mode();
4672 :
4673 : pte -= (addr - start) / PAGE_SIZE;
4674 :
4675 : for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
4676 : unsigned long pfn;
4677 :
4678 : pfn = get_pte_pfn(pte[i], pvmw->vma, addr);
4679 : if (pfn == -1)
4680 : continue;
4681 :
4682 : if (!pte_young(pte[i]))
4683 : continue;
4684 :
4685 : folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap);
4686 : if (!folio)
4687 : continue;
4688 :
4689 : if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i))
4690 : VM_WARN_ON_ONCE(true);
4691 :
4692 : young++;
4693 :
4694 : if (pte_dirty(pte[i]) && !folio_test_dirty(folio) &&
4695 : !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
4696 : !folio_test_swapcache(folio)))
4697 : folio_mark_dirty(folio);
4698 :
4699 : if (walk) {
4700 : old_gen = folio_update_gen(folio, new_gen);
4701 : if (old_gen >= 0 && old_gen != new_gen)
4702 : update_batch_size(walk, folio, old_gen, new_gen);
4703 :
4704 : continue;
4705 : }
4706 :
4707 : old_gen = folio_lru_gen(folio);
4708 : if (old_gen < 0)
4709 : folio_set_referenced(folio);
4710 : else if (old_gen != new_gen)
4711 : folio_activate(folio);
4712 : }
4713 :
4714 : arch_leave_lazy_mmu_mode();
4715 : mem_cgroup_unlock_pages();
4716 :
4717 : /* feedback from rmap walkers to page table walkers */
4718 : if (suitable_to_scan(i, young))
4719 : update_bloom_filter(lruvec, max_seq, pvmw->pmd);
4720 : }
4721 :
4722 : /******************************************************************************
4723 : * memcg LRU
4724 : ******************************************************************************/
4725 :
4726 : /* see the comment on MEMCG_NR_GENS */
4727 : enum {
4728 : MEMCG_LRU_NOP,
4729 : MEMCG_LRU_HEAD,
4730 : MEMCG_LRU_TAIL,
4731 : MEMCG_LRU_OLD,
4732 : MEMCG_LRU_YOUNG,
4733 : };
4734 :
4735 : #ifdef CONFIG_MEMCG
4736 :
4737 : static int lru_gen_memcg_seg(struct lruvec *lruvec)
4738 : {
4739 : return READ_ONCE(lruvec->lrugen.seg);
4740 : }
4741 :
4742 : static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
4743 : {
4744 : int seg;
4745 : int old, new;
4746 : int bin = get_random_u32_below(MEMCG_NR_BINS);
4747 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
4748 :
4749 : spin_lock(&pgdat->memcg_lru.lock);
4750 :
4751 : VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
4752 :
4753 : seg = 0;
4754 : new = old = lruvec->lrugen.gen;
4755 :
4756 : /* see the comment on MEMCG_NR_GENS */
4757 : if (op == MEMCG_LRU_HEAD)
4758 : seg = MEMCG_LRU_HEAD;
4759 : else if (op == MEMCG_LRU_TAIL)
4760 : seg = MEMCG_LRU_TAIL;
4761 : else if (op == MEMCG_LRU_OLD)
4762 : new = get_memcg_gen(pgdat->memcg_lru.seq);
4763 : else if (op == MEMCG_LRU_YOUNG)
4764 : new = get_memcg_gen(pgdat->memcg_lru.seq + 1);
4765 : else
4766 : VM_WARN_ON_ONCE(true);
4767 :
4768 : hlist_nulls_del_rcu(&lruvec->lrugen.list);
4769 :
4770 : if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD)
4771 : hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4772 : else
4773 : hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4774 :
4775 : pgdat->memcg_lru.nr_memcgs[old]--;
4776 : pgdat->memcg_lru.nr_memcgs[new]++;
4777 :
4778 : lruvec->lrugen.gen = new;
4779 : WRITE_ONCE(lruvec->lrugen.seg, seg);
4780 :
4781 : if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq))
4782 : WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
4783 :
4784 : spin_unlock(&pgdat->memcg_lru.lock);
4785 : }
4786 :
4787 : void lru_gen_online_memcg(struct mem_cgroup *memcg)
4788 : {
4789 : int gen;
4790 : int nid;
4791 : int bin = get_random_u32_below(MEMCG_NR_BINS);
4792 :
4793 : for_each_node(nid) {
4794 : struct pglist_data *pgdat = NODE_DATA(nid);
4795 : struct lruvec *lruvec = get_lruvec(memcg, nid);
4796 :
4797 : spin_lock(&pgdat->memcg_lru.lock);
4798 :
4799 : VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
4800 :
4801 : gen = get_memcg_gen(pgdat->memcg_lru.seq);
4802 :
4803 : hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
4804 : pgdat->memcg_lru.nr_memcgs[gen]++;
4805 :
4806 : lruvec->lrugen.gen = gen;
4807 :
4808 : spin_unlock(&pgdat->memcg_lru.lock);
4809 : }
4810 : }
4811 :
4812 : void lru_gen_offline_memcg(struct mem_cgroup *memcg)
4813 : {
4814 : int nid;
4815 :
4816 : for_each_node(nid) {
4817 : struct lruvec *lruvec = get_lruvec(memcg, nid);
4818 :
4819 : lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD);
4820 : }
4821 : }
4822 :
4823 : void lru_gen_release_memcg(struct mem_cgroup *memcg)
4824 : {
4825 : int gen;
4826 : int nid;
4827 :
4828 : for_each_node(nid) {
4829 : struct pglist_data *pgdat = NODE_DATA(nid);
4830 : struct lruvec *lruvec = get_lruvec(memcg, nid);
4831 :
4832 : spin_lock(&pgdat->memcg_lru.lock);
4833 :
4834 : VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
4835 :
4836 : gen = lruvec->lrugen.gen;
4837 :
4838 : hlist_nulls_del_rcu(&lruvec->lrugen.list);
4839 : pgdat->memcg_lru.nr_memcgs[gen]--;
4840 :
4841 : if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq))
4842 : WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
4843 :
4844 : spin_unlock(&pgdat->memcg_lru.lock);
4845 : }
4846 : }
4847 :
4848 : void lru_gen_soft_reclaim(struct lruvec *lruvec)
4849 : {
4850 : /* see the comment on MEMCG_NR_GENS */
4851 : if (lru_gen_memcg_seg(lruvec) != MEMCG_LRU_HEAD)
4852 : lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
4853 : }
4854 :
4855 : #else /* !CONFIG_MEMCG */
4856 :
4857 : static int lru_gen_memcg_seg(struct lruvec *lruvec)
4858 : {
4859 : return 0;
4860 : }
4861 :
4862 : #endif
4863 :
4864 : /******************************************************************************
4865 : * the eviction
4866 : ******************************************************************************/
4867 :
4868 : static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
4869 : {
4870 : bool success;
4871 : int gen = folio_lru_gen(folio);
4872 : int type = folio_is_file_lru(folio);
4873 : int zone = folio_zonenum(folio);
4874 : int delta = folio_nr_pages(folio);
4875 : int refs = folio_lru_refs(folio);
4876 : int tier = lru_tier_from_refs(refs);
4877 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
4878 :
4879 : VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
4880 :
4881 : /* unevictable */
4882 : if (!folio_evictable(folio)) {
4883 : success = lru_gen_del_folio(lruvec, folio, true);
4884 : VM_WARN_ON_ONCE_FOLIO(!success, folio);
4885 : folio_set_unevictable(folio);
4886 : lruvec_add_folio(lruvec, folio);
4887 : __count_vm_events(UNEVICTABLE_PGCULLED, delta);
4888 : return true;
4889 : }
4890 :
4891 : /* dirty lazyfree */
4892 : if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) {
4893 : success = lru_gen_del_folio(lruvec, folio, true);
4894 : VM_WARN_ON_ONCE_FOLIO(!success, folio);
4895 : folio_set_swapbacked(folio);
4896 : lruvec_add_folio_tail(lruvec, folio);
4897 : return true;
4898 : }
4899 :
4900 : /* promoted */
4901 : if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
4902 : list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4903 : return true;
4904 : }
4905 :
4906 : /* protected */
4907 : if (tier > tier_idx) {
4908 : int hist = lru_hist_from_seq(lrugen->min_seq[type]);
4909 :
4910 : gen = folio_inc_gen(lruvec, folio, false);
4911 : list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
4912 :
4913 : WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
4914 : lrugen->protected[hist][type][tier - 1] + delta);
4915 : __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
4916 : return true;
4917 : }
4918 :
4919 : /* waiting for writeback */
4920 : if (folio_test_locked(folio) || folio_test_writeback(folio) ||
4921 : (type == LRU_GEN_FILE && folio_test_dirty(folio))) {
4922 : gen = folio_inc_gen(lruvec, folio, true);
4923 : list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4924 : return true;
4925 : }
4926 :
4927 : return false;
4928 : }
4929 :
4930 : static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc)
4931 : {
4932 : bool success;
4933 :
4934 : /* swapping inhibited */
4935 : if (!(sc->gfp_mask & __GFP_IO) &&
4936 : (folio_test_dirty(folio) ||
4937 : (folio_test_anon(folio) && !folio_test_swapcache(folio))))
4938 : return false;
4939 :
4940 : /* raced with release_pages() */
4941 : if (!folio_try_get(folio))
4942 : return false;
4943 :
4944 : /* raced with another isolation */
4945 : if (!folio_test_clear_lru(folio)) {
4946 : folio_put(folio);
4947 : return false;
4948 : }
4949 :
4950 : /* see the comment on MAX_NR_TIERS */
4951 : if (!folio_test_referenced(folio))
4952 : set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);
4953 :
4954 : /* for shrink_folio_list() */
4955 : folio_clear_reclaim(folio);
4956 : folio_clear_referenced(folio);
4957 :
4958 : success = lru_gen_del_folio(lruvec, folio, true);
4959 : VM_WARN_ON_ONCE_FOLIO(!success, folio);
4960 :
4961 : return true;
4962 : }
4963 :
4964 : static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
4965 : int type, int tier, struct list_head *list)
4966 : {
4967 : int gen, zone;
4968 : enum vm_event_item item;
4969 : int sorted = 0;
4970 : int scanned = 0;
4971 : int isolated = 0;
4972 : int remaining = MAX_LRU_BATCH;
4973 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
4974 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4975 :
4976 : VM_WARN_ON_ONCE(!list_empty(list));
4977 :
4978 : if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
4979 : return 0;
4980 :
4981 : gen = lru_gen_from_seq(lrugen->min_seq[type]);
4982 :
4983 : for (zone = sc->reclaim_idx; zone >= 0; zone--) {
4984 : LIST_HEAD(moved);
4985 : int skipped = 0;
4986 : struct list_head *head = &lrugen->folios[gen][type][zone];
4987 :
4988 : while (!list_empty(head)) {
4989 : struct folio *folio = lru_to_folio(head);
4990 : int delta = folio_nr_pages(folio);
4991 :
4992 : VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
4993 : VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
4994 : VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
4995 : VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
4996 :
4997 : scanned += delta;
4998 :
4999 : if (sort_folio(lruvec, folio, tier))
5000 : sorted += delta;
5001 : else if (isolate_folio(lruvec, folio, sc)) {
5002 : list_add(&folio->lru, list);
5003 : isolated += delta;
5004 : } else {
5005 : list_move(&folio->lru, &moved);
5006 : skipped += delta;
5007 : }
5008 :
5009 : if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH)
5010 : break;
5011 : }
5012 :
5013 : if (skipped) {
5014 : list_splice(&moved, head);
5015 : __count_zid_vm_events(PGSCAN_SKIP, zone, skipped);
5016 : }
5017 :
5018 : if (!remaining || isolated >= MIN_LRU_BATCH)
5019 : break;
5020 : }
5021 :
5022 : item = PGSCAN_KSWAPD + reclaimer_offset();
5023 : if (!cgroup_reclaim(sc)) {
5024 : __count_vm_events(item, isolated);
5025 : __count_vm_events(PGREFILL, sorted);
5026 : }
5027 : __count_memcg_events(memcg, item, isolated);
5028 : __count_memcg_events(memcg, PGREFILL, sorted);
5029 : __count_vm_events(PGSCAN_ANON + type, isolated);
5030 :
5031 : /*
5032 : * There might not be eligible folios due to reclaim_idx. Check the
5033 : * remaining to prevent livelock if it's not making progress.
5034 : */
5035 : return isolated || !remaining ? scanned : 0;
5036 : }
5037 :
5038 : static int get_tier_idx(struct lruvec *lruvec, int type)
5039 : {
5040 : int tier;
5041 : struct ctrl_pos sp, pv;
5042 :
5043 : /*
5044 : * To leave a margin for fluctuations, use a larger gain factor (1:2).
5045 : * This value is chosen because any other tier would have at least twice
5046 : * as many refaults as the first tier.
5047 : */
5048 : read_ctrl_pos(lruvec, type, 0, 1, &sp);
5049 : for (tier = 1; tier < MAX_NR_TIERS; tier++) {
5050 : read_ctrl_pos(lruvec, type, tier, 2, &pv);
5051 : if (!positive_ctrl_err(&sp, &pv))
5052 : break;
5053 : }
5054 :
5055 : return tier - 1;
5056 : }
5057 :
5058 : static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx)
5059 : {
5060 : int type, tier;
5061 : struct ctrl_pos sp, pv;
5062 : int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness };
5063 :
5064 : /*
5065 : * Compare the first tier of anon with that of file to determine which
5066 : * type to scan. Also need to compare other tiers of the selected type
5067 : * with the first tier of the other type to determine the last tier (of
5068 : * the selected type) to evict.
5069 : */
5070 : read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp);
5071 : read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv);
5072 : type = positive_ctrl_err(&sp, &pv);
5073 :
5074 : read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp);
5075 : for (tier = 1; tier < MAX_NR_TIERS; tier++) {
5076 : read_ctrl_pos(lruvec, type, tier, gain[type], &pv);
5077 : if (!positive_ctrl_err(&sp, &pv))
5078 : break;
5079 : }
5080 :
5081 : *tier_idx = tier - 1;
5082 :
5083 : return type;
5084 : }
5085 :
5086 : static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness,
5087 : int *type_scanned, struct list_head *list)
5088 : {
5089 : int i;
5090 : int type;
5091 : int scanned;
5092 : int tier = -1;
5093 : DEFINE_MIN_SEQ(lruvec);
5094 :
5095 : /*
5096 : * Try to make the obvious choice first. When anon and file are both
5097 : * available from the same generation, interpret swappiness 1 as file
5098 : * first and 200 as anon first.
5099 : */
5100 : if (!swappiness)
5101 : type = LRU_GEN_FILE;
5102 : else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE])
5103 : type = LRU_GEN_ANON;
5104 : else if (swappiness == 1)
5105 : type = LRU_GEN_FILE;
5106 : else if (swappiness == 200)
5107 : type = LRU_GEN_ANON;
5108 : else
5109 : type = get_type_to_scan(lruvec, swappiness, &tier);
5110 :
5111 : for (i = !swappiness; i < ANON_AND_FILE; i++) {
5112 : if (tier < 0)
5113 : tier = get_tier_idx(lruvec, type);
5114 :
5115 : scanned = scan_folios(lruvec, sc, type, tier, list);
5116 : if (scanned)
5117 : break;
5118 :
5119 : type = !type;
5120 : tier = -1;
5121 : }
5122 :
5123 : *type_scanned = type;
5124 :
5125 : return scanned;
5126 : }
5127 :
5128 : static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness)
5129 : {
5130 : int type;
5131 : int scanned;
5132 : int reclaimed;
5133 : LIST_HEAD(list);
5134 : LIST_HEAD(clean);
5135 : struct folio *folio;
5136 : struct folio *next;
5137 : enum vm_event_item item;
5138 : struct reclaim_stat stat;
5139 : struct lru_gen_mm_walk *walk;
5140 : bool skip_retry = false;
5141 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5142 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
5143 :
5144 : spin_lock_irq(&lruvec->lru_lock);
5145 :
5146 : scanned = isolate_folios(lruvec, sc, swappiness, &type, &list);
5147 :
5148 : scanned += try_to_inc_min_seq(lruvec, swappiness);
5149 :
5150 : if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS)
5151 : scanned = 0;
5152 :
5153 : spin_unlock_irq(&lruvec->lru_lock);
5154 :
5155 : if (list_empty(&list))
5156 : return scanned;
5157 : retry:
5158 : reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false);
5159 : sc->nr_reclaimed += reclaimed;
5160 :
5161 : list_for_each_entry_safe_reverse(folio, next, &list, lru) {
5162 : if (!folio_evictable(folio)) {
5163 : list_del(&folio->lru);
5164 : folio_putback_lru(folio);
5165 : continue;
5166 : }
5167 :
5168 : if (folio_test_reclaim(folio) &&
5169 : (folio_test_dirty(folio) || folio_test_writeback(folio))) {
5170 : /* restore LRU_REFS_FLAGS cleared by isolate_folio() */
5171 : if (folio_test_workingset(folio))
5172 : folio_set_referenced(folio);
5173 : continue;
5174 : }
5175 :
5176 : if (skip_retry || folio_test_active(folio) || folio_test_referenced(folio) ||
5177 : folio_mapped(folio) || folio_test_locked(folio) ||
5178 : folio_test_dirty(folio) || folio_test_writeback(folio)) {
5179 : /* don't add rejected folios to the oldest generation */
5180 : set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS,
5181 : BIT(PG_active));
5182 : continue;
5183 : }
5184 :
5185 : /* retry folios that may have missed folio_rotate_reclaimable() */
5186 : list_move(&folio->lru, &clean);
5187 : sc->nr_scanned -= folio_nr_pages(folio);
5188 : }
5189 :
5190 : spin_lock_irq(&lruvec->lru_lock);
5191 :
5192 : move_folios_to_lru(lruvec, &list);
5193 :
5194 : walk = current->reclaim_state->mm_walk;
5195 : if (walk && walk->batched)
5196 : reset_batch_size(lruvec, walk);
5197 :
5198 : item = PGSTEAL_KSWAPD + reclaimer_offset();
5199 : if (!cgroup_reclaim(sc))
5200 : __count_vm_events(item, reclaimed);
5201 : __count_memcg_events(memcg, item, reclaimed);
5202 : __count_vm_events(PGSTEAL_ANON + type, reclaimed);
5203 :
5204 : spin_unlock_irq(&lruvec->lru_lock);
5205 :
5206 : mem_cgroup_uncharge_list(&list);
5207 : free_unref_page_list(&list);
5208 :
5209 : INIT_LIST_HEAD(&list);
5210 : list_splice_init(&clean, &list);
5211 :
5212 : if (!list_empty(&list)) {
5213 : skip_retry = true;
5214 : goto retry;
5215 : }
5216 :
5217 : return scanned;
5218 : }
5219 :
5220 : static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
5221 : struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
5222 : {
5223 : int gen, type, zone;
5224 : unsigned long old = 0;
5225 : unsigned long young = 0;
5226 : unsigned long total = 0;
5227 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
5228 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5229 : DEFINE_MIN_SEQ(lruvec);
5230 :
5231 : /* whether this lruvec is completely out of cold folios */
5232 : if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) {
5233 : *nr_to_scan = 0;
5234 : return true;
5235 : }
5236 :
5237 : for (type = !can_swap; type < ANON_AND_FILE; type++) {
5238 : unsigned long seq;
5239 :
5240 : for (seq = min_seq[type]; seq <= max_seq; seq++) {
5241 : unsigned long size = 0;
5242 :
5243 : gen = lru_gen_from_seq(seq);
5244 :
5245 : for (zone = 0; zone < MAX_NR_ZONES; zone++)
5246 : size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
5247 :
5248 : total += size;
5249 : if (seq == max_seq)
5250 : young += size;
5251 : else if (seq + MIN_NR_GENS == max_seq)
5252 : old += size;
5253 : }
5254 : }
5255 :
5256 : /* try to scrape all its memory if this memcg was deleted */
5257 : *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
5258 :
5259 : /*
5260 : * The aging tries to be lazy to reduce the overhead, while the eviction
5261 : * stalls when the number of generations reaches MIN_NR_GENS. Hence, the
5262 : * ideal number of generations is MIN_NR_GENS+1.
5263 : */
5264 : if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
5265 : return false;
5266 :
5267 : /*
5268 : * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1)
5269 : * of the total number of pages for each generation. A reasonable range
5270 : * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The
5271 : * aging cares about the upper bound of hot pages, while the eviction
5272 : * cares about the lower bound of cold pages.
5273 : */
5274 : if (young * MIN_NR_GENS > total)
5275 : return true;
5276 : if (old * (MIN_NR_GENS + 2) < total)
5277 : return true;
5278 :
5279 : return false;
5280 : }
5281 :
5282 : /*
5283 : * For future optimizations:
5284 : * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
5285 : * reclaim.
5286 : */
5287 : static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap)
5288 : {
5289 : unsigned long nr_to_scan;
5290 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5291 : DEFINE_MAX_SEQ(lruvec);
5292 :
5293 : if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg))
5294 : return 0;
5295 :
5296 : if (!should_run_aging(lruvec, max_seq, sc, can_swap, &nr_to_scan))
5297 : return nr_to_scan;
5298 :
5299 : /* skip the aging path at the default priority */
5300 : if (sc->priority == DEF_PRIORITY)
5301 : return nr_to_scan;
5302 :
5303 : /* skip this lruvec as it's low on cold folios */
5304 : return try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false) ? -1 : 0;
5305 : }
5306 :
5307 : static unsigned long get_nr_to_reclaim(struct scan_control *sc)
5308 : {
5309 : /* don't abort memcg reclaim to ensure fairness */
5310 : if (!global_reclaim(sc))
5311 : return -1;
5312 :
5313 : return max(sc->nr_to_reclaim, compact_gap(sc->order));
5314 : }
5315 :
5316 : static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5317 : {
5318 : long nr_to_scan;
5319 : unsigned long scanned = 0;
5320 : unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
5321 : int swappiness = get_swappiness(lruvec, sc);
5322 :
5323 : /* clean file folios are more likely to exist */
5324 : if (swappiness && !(sc->gfp_mask & __GFP_IO))
5325 : swappiness = 1;
5326 :
5327 : while (true) {
5328 : int delta;
5329 :
5330 : nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
5331 : if (nr_to_scan <= 0)
5332 : break;
5333 :
5334 : delta = evict_folios(lruvec, sc, swappiness);
5335 : if (!delta)
5336 : break;
5337 :
5338 : scanned += delta;
5339 : if (scanned >= nr_to_scan)
5340 : break;
5341 :
5342 : if (sc->nr_reclaimed >= nr_to_reclaim)
5343 : break;
5344 :
5345 : cond_resched();
5346 : }
5347 :
5348 : /* whether try_to_inc_max_seq() was successful */
5349 : return nr_to_scan < 0;
5350 : }
5351 :
5352 : static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
5353 : {
5354 : bool success;
5355 : unsigned long scanned = sc->nr_scanned;
5356 : unsigned long reclaimed = sc->nr_reclaimed;
5357 : int seg = lru_gen_memcg_seg(lruvec);
5358 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5359 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
5360 :
5361 : /* see the comment on MEMCG_NR_GENS */
5362 : if (!lruvec_is_sizable(lruvec, sc))
5363 : return seg != MEMCG_LRU_TAIL ? MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG;
5364 :
5365 : mem_cgroup_calculate_protection(NULL, memcg);
5366 :
5367 : if (mem_cgroup_below_min(NULL, memcg))
5368 : return MEMCG_LRU_YOUNG;
5369 :
5370 : if (mem_cgroup_below_low(NULL, memcg)) {
5371 : /* see the comment on MEMCG_NR_GENS */
5372 : if (seg != MEMCG_LRU_TAIL)
5373 : return MEMCG_LRU_TAIL;
5374 :
5375 : memcg_memory_event(memcg, MEMCG_LOW);
5376 : }
5377 :
5378 : success = try_to_shrink_lruvec(lruvec, sc);
5379 :
5380 : shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);
5381 :
5382 : if (!sc->proactive)
5383 : vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned,
5384 : sc->nr_reclaimed - reclaimed);
5385 :
5386 : flush_reclaim_state(sc);
5387 :
5388 : return success ? MEMCG_LRU_YOUNG : 0;
5389 : }
5390 :
5391 : #ifdef CONFIG_MEMCG
5392 :
5393 : static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
5394 : {
5395 : int op;
5396 : int gen;
5397 : int bin;
5398 : int first_bin;
5399 : struct lruvec *lruvec;
5400 : struct lru_gen_folio *lrugen;
5401 : struct mem_cgroup *memcg;
5402 : const struct hlist_nulls_node *pos;
5403 : unsigned long nr_to_reclaim = get_nr_to_reclaim(sc);
5404 :
5405 : bin = first_bin = get_random_u32_below(MEMCG_NR_BINS);
5406 : restart:
5407 : op = 0;
5408 : memcg = NULL;
5409 : gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq));
5410 :
5411 : rcu_read_lock();
5412 :
5413 : hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {
5414 : if (op)
5415 : lru_gen_rotate_memcg(lruvec, op);
5416 :
5417 : mem_cgroup_put(memcg);
5418 :
5419 : lruvec = container_of(lrugen, struct lruvec, lrugen);
5420 : memcg = lruvec_memcg(lruvec);
5421 :
5422 : if (!mem_cgroup_tryget(memcg)) {
5423 : op = 0;
5424 : memcg = NULL;
5425 : continue;
5426 : }
5427 :
5428 : rcu_read_unlock();
5429 :
5430 : op = shrink_one(lruvec, sc);
5431 :
5432 : rcu_read_lock();
5433 :
5434 : if (sc->nr_reclaimed >= nr_to_reclaim)
5435 : break;
5436 : }
5437 :
5438 : rcu_read_unlock();
5439 :
5440 : if (op)
5441 : lru_gen_rotate_memcg(lruvec, op);
5442 :
5443 : mem_cgroup_put(memcg);
5444 :
5445 : if (sc->nr_reclaimed >= nr_to_reclaim)
5446 : return;
5447 :
5448 : /* restart if raced with lru_gen_rotate_memcg() */
5449 : if (gen != get_nulls_value(pos))
5450 : goto restart;
5451 :
5452 : /* try the rest of the bins of the current generation */
5453 : bin = get_memcg_bin(bin + 1);
5454 : if (bin != first_bin)
5455 : goto restart;
5456 : }
5457 :
5458 : static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5459 : {
5460 : struct blk_plug plug;
5461 :
5462 : VM_WARN_ON_ONCE(global_reclaim(sc));
5463 : VM_WARN_ON_ONCE(!sc->may_writepage || !sc->may_unmap);
5464 :
5465 : lru_add_drain();
5466 :
5467 : blk_start_plug(&plug);
5468 :
5469 : set_mm_walk(NULL, sc->proactive);
5470 :
5471 : if (try_to_shrink_lruvec(lruvec, sc))
5472 : lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
5473 :
5474 : clear_mm_walk();
5475 :
5476 : blk_finish_plug(&plug);
5477 : }
5478 :
5479 : #else /* !CONFIG_MEMCG */
5480 :
5481 : static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc)
5482 : {
5483 : BUILD_BUG();
5484 : }
5485 :
5486 : static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
5487 : {
5488 : BUILD_BUG();
5489 : }
5490 :
5491 : #endif
5492 :
5493 : static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc)
5494 : {
5495 : int priority;
5496 : unsigned long reclaimable;
5497 : struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
5498 :
5499 : if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH)
5500 : return;
5501 : /*
5502 : * Determine the initial priority based on ((total / MEMCG_NR_GENS) >>
5503 : * priority) * reclaimed_to_scanned_ratio = nr_to_reclaim, where the
5504 : * estimated reclaimed_to_scanned_ratio = inactive / total.
5505 : */
5506 : reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE);
5507 : if (get_swappiness(lruvec, sc))
5508 : reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON);
5509 :
5510 : reclaimable /= MEMCG_NR_GENS;
5511 :
5512 : /* round down reclaimable and round up sc->nr_to_reclaim */
5513 : priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1);
5514 :
5515 : sc->priority = clamp(priority, 0, DEF_PRIORITY);
5516 : }
5517 :
5518 : static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
5519 : {
5520 : struct blk_plug plug;
5521 : unsigned long reclaimed = sc->nr_reclaimed;
5522 :
5523 : VM_WARN_ON_ONCE(!global_reclaim(sc));
5524 :
5525 : /*
5526 : * Unmapped clean folios are already prioritized. Scanning for more of
5527 : * them is likely futile and can cause high reclaim latency when there
5528 : * is a large number of memcgs.
5529 : */
5530 : if (!sc->may_writepage || !sc->may_unmap)
5531 : goto done;
5532 :
5533 : lru_add_drain();
5534 :
5535 : blk_start_plug(&plug);
5536 :
5537 : set_mm_walk(pgdat, sc->proactive);
5538 :
5539 : set_initial_priority(pgdat, sc);
5540 :
5541 : if (current_is_kswapd())
5542 : sc->nr_reclaimed = 0;
5543 :
5544 : if (mem_cgroup_disabled())
5545 : shrink_one(&pgdat->__lruvec, sc);
5546 : else
5547 : shrink_many(pgdat, sc);
5548 :
5549 : if (current_is_kswapd())
5550 : sc->nr_reclaimed += reclaimed;
5551 :
5552 : clear_mm_walk();
5553 :
5554 : blk_finish_plug(&plug);
5555 : done:
5556 : /* kswapd should never fail */
5557 : pgdat->kswapd_failures = 0;
5558 : }
5559 :
5560 : /******************************************************************************
5561 : * state change
5562 : ******************************************************************************/
5563 :
5564 : static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
5565 : {
5566 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
5567 :
5568 : if (lrugen->enabled) {
5569 : enum lru_list lru;
5570 :
5571 : for_each_evictable_lru(lru) {
5572 : if (!list_empty(&lruvec->lists[lru]))
5573 : return false;
5574 : }
5575 : } else {
5576 : int gen, type, zone;
5577 :
5578 : for_each_gen_type_zone(gen, type, zone) {
5579 : if (!list_empty(&lrugen->folios[gen][type][zone]))
5580 : return false;
5581 : }
5582 : }
5583 :
5584 : return true;
5585 : }
5586 :
5587 : static bool fill_evictable(struct lruvec *lruvec)
5588 : {
5589 : enum lru_list lru;
5590 : int remaining = MAX_LRU_BATCH;
5591 :
5592 : for_each_evictable_lru(lru) {
5593 : int type = is_file_lru(lru);
5594 : bool active = is_active_lru(lru);
5595 : struct list_head *head = &lruvec->lists[lru];
5596 :
5597 : while (!list_empty(head)) {
5598 : bool success;
5599 : struct folio *folio = lru_to_folio(head);
5600 :
5601 : VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
5602 : VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio);
5603 : VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
5604 : VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio);
5605 :
5606 : lruvec_del_folio(lruvec, folio);
5607 : success = lru_gen_add_folio(lruvec, folio, false);
5608 : VM_WARN_ON_ONCE(!success);
5609 :
5610 : if (!--remaining)
5611 : return false;
5612 : }
5613 : }
5614 :
5615 : return true;
5616 : }
5617 :
5618 : static bool drain_evictable(struct lruvec *lruvec)
5619 : {
5620 : int gen, type, zone;
5621 : int remaining = MAX_LRU_BATCH;
5622 :
5623 : for_each_gen_type_zone(gen, type, zone) {
5624 : struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
5625 :
5626 : while (!list_empty(head)) {
5627 : bool success;
5628 : struct folio *folio = lru_to_folio(head);
5629 :
5630 : VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
5631 : VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
5632 : VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
5633 : VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
5634 :
5635 : success = lru_gen_del_folio(lruvec, folio, false);
5636 : VM_WARN_ON_ONCE(!success);
5637 : lruvec_add_folio(lruvec, folio);
5638 :
5639 : if (!--remaining)
5640 : return false;
5641 : }
5642 : }
5643 :
5644 : return true;
5645 : }
5646 :
5647 : static void lru_gen_change_state(bool enabled)
5648 : {
5649 : static DEFINE_MUTEX(state_mutex);
5650 :
5651 : struct mem_cgroup *memcg;
5652 :
5653 : cgroup_lock();
5654 : cpus_read_lock();
5655 : get_online_mems();
5656 : mutex_lock(&state_mutex);
5657 :
5658 : if (enabled == lru_gen_enabled())
5659 : goto unlock;
5660 :
5661 : if (enabled)
5662 : static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
5663 : else
5664 : static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
5665 :
5666 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
5667 : do {
5668 : int nid;
5669 :
5670 : for_each_node(nid) {
5671 : struct lruvec *lruvec = get_lruvec(memcg, nid);
5672 :
5673 : spin_lock_irq(&lruvec->lru_lock);
5674 :
5675 : VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
5676 : VM_WARN_ON_ONCE(!state_is_valid(lruvec));
5677 :
5678 : lruvec->lrugen.enabled = enabled;
5679 :
5680 : while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) {
5681 : spin_unlock_irq(&lruvec->lru_lock);
5682 : cond_resched();
5683 : spin_lock_irq(&lruvec->lru_lock);
5684 : }
5685 :
5686 : spin_unlock_irq(&lruvec->lru_lock);
5687 : }
5688 :
5689 : cond_resched();
5690 : } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
5691 : unlock:
5692 : mutex_unlock(&state_mutex);
5693 : put_online_mems();
5694 : cpus_read_unlock();
5695 : cgroup_unlock();
5696 : }
5697 :
5698 : /******************************************************************************
5699 : * sysfs interface
5700 : ******************************************************************************/
5701 :
5702 : static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
5703 : {
5704 : return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl)));
5705 : }
5706 :
5707 : /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5708 : static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr,
5709 : const char *buf, size_t len)
5710 : {
5711 : unsigned int msecs;
5712 :
5713 : if (kstrtouint(buf, 0, &msecs))
5714 : return -EINVAL;
5715 :
5716 : WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs));
5717 :
5718 : return len;
5719 : }
5720 :
5721 : static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms);
5722 :
5723 : static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
5724 : {
5725 : unsigned int caps = 0;
5726 :
5727 : if (get_cap(LRU_GEN_CORE))
5728 : caps |= BIT(LRU_GEN_CORE);
5729 :
5730 : if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))
5731 : caps |= BIT(LRU_GEN_MM_WALK);
5732 :
5733 : if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
5734 : caps |= BIT(LRU_GEN_NONLEAF_YOUNG);
5735 :
5736 : return sysfs_emit(buf, "0x%04x\n", caps);
5737 : }
5738 :
5739 : /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5740 : static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
5741 : const char *buf, size_t len)
5742 : {
5743 : int i;
5744 : unsigned int caps;
5745 :
5746 : if (tolower(*buf) == 'n')
5747 : caps = 0;
5748 : else if (tolower(*buf) == 'y')
5749 : caps = -1;
5750 : else if (kstrtouint(buf, 0, &caps))
5751 : return -EINVAL;
5752 :
5753 : for (i = 0; i < NR_LRU_GEN_CAPS; i++) {
5754 : bool enabled = caps & BIT(i);
5755 :
5756 : if (i == LRU_GEN_CORE)
5757 : lru_gen_change_state(enabled);
5758 : else if (enabled)
5759 : static_branch_enable(&lru_gen_caps[i]);
5760 : else
5761 : static_branch_disable(&lru_gen_caps[i]);
5762 : }
5763 :
5764 : return len;
5765 : }
5766 :
5767 : static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled);
5768 :
5769 : static struct attribute *lru_gen_attrs[] = {
5770 : &lru_gen_min_ttl_attr.attr,
5771 : &lru_gen_enabled_attr.attr,
5772 : NULL
5773 : };
5774 :
5775 : static const struct attribute_group lru_gen_attr_group = {
5776 : .name = "lru_gen",
5777 : .attrs = lru_gen_attrs,
5778 : };
5779 :
5780 : /******************************************************************************
5781 : * debugfs interface
5782 : ******************************************************************************/
5783 :
5784 : static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos)
5785 : {
5786 : struct mem_cgroup *memcg;
5787 : loff_t nr_to_skip = *pos;
5788 :
5789 : m->private = kvmalloc(PATH_MAX, GFP_KERNEL);
5790 : if (!m->private)
5791 : return ERR_PTR(-ENOMEM);
5792 :
5793 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
5794 : do {
5795 : int nid;
5796 :
5797 : for_each_node_state(nid, N_MEMORY) {
5798 : if (!nr_to_skip--)
5799 : return get_lruvec(memcg, nid);
5800 : }
5801 : } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
5802 :
5803 : return NULL;
5804 : }
5805 :
5806 : static void lru_gen_seq_stop(struct seq_file *m, void *v)
5807 : {
5808 : if (!IS_ERR_OR_NULL(v))
5809 : mem_cgroup_iter_break(NULL, lruvec_memcg(v));
5810 :
5811 : kvfree(m->private);
5812 : m->private = NULL;
5813 : }
5814 :
5815 : static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos)
5816 : {
5817 : int nid = lruvec_pgdat(v)->node_id;
5818 : struct mem_cgroup *memcg = lruvec_memcg(v);
5819 :
5820 : ++*pos;
5821 :
5822 : nid = next_memory_node(nid);
5823 : if (nid == MAX_NUMNODES) {
5824 : memcg = mem_cgroup_iter(NULL, memcg, NULL);
5825 : if (!memcg)
5826 : return NULL;
5827 :
5828 : nid = first_memory_node;
5829 : }
5830 :
5831 : return get_lruvec(memcg, nid);
5832 : }
5833 :
5834 : static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
5835 : unsigned long max_seq, unsigned long *min_seq,
5836 : unsigned long seq)
5837 : {
5838 : int i;
5839 : int type, tier;
5840 : int hist = lru_hist_from_seq(seq);
5841 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
5842 :
5843 : for (tier = 0; tier < MAX_NR_TIERS; tier++) {
5844 : seq_printf(m, " %10d", tier);
5845 : for (type = 0; type < ANON_AND_FILE; type++) {
5846 : const char *s = " ";
5847 : unsigned long n[3] = {};
5848 :
5849 : if (seq == max_seq) {
5850 : s = "RT ";
5851 : n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]);
5852 : n[1] = READ_ONCE(lrugen->avg_total[type][tier]);
5853 : } else if (seq == min_seq[type] || NR_HIST_GENS > 1) {
5854 : s = "rep";
5855 : n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]);
5856 : n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]);
5857 : if (tier)
5858 : n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]);
5859 : }
5860 :
5861 : for (i = 0; i < 3; i++)
5862 : seq_printf(m, " %10lu%c", n[i], s[i]);
5863 : }
5864 : seq_putc(m, '\n');
5865 : }
5866 :
5867 : seq_puts(m, " ");
5868 : for (i = 0; i < NR_MM_STATS; i++) {
5869 : const char *s = " ";
5870 : unsigned long n = 0;
5871 :
5872 : if (seq == max_seq && NR_HIST_GENS == 1) {
5873 : s = "LOYNFA";
5874 : n = READ_ONCE(lruvec->mm_state.stats[hist][i]);
5875 : } else if (seq != max_seq && NR_HIST_GENS > 1) {
5876 : s = "loynfa";
5877 : n = READ_ONCE(lruvec->mm_state.stats[hist][i]);
5878 : }
5879 :
5880 : seq_printf(m, " %10lu%c", n, s[i]);
5881 : }
5882 : seq_putc(m, '\n');
5883 : }
5884 :
5885 : /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
5886 : static int lru_gen_seq_show(struct seq_file *m, void *v)
5887 : {
5888 : unsigned long seq;
5889 : bool full = !debugfs_real_fops(m->file)->write;
5890 : struct lruvec *lruvec = v;
5891 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
5892 : int nid = lruvec_pgdat(lruvec)->node_id;
5893 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
5894 : DEFINE_MAX_SEQ(lruvec);
5895 : DEFINE_MIN_SEQ(lruvec);
5896 :
5897 : if (nid == first_memory_node) {
5898 : const char *path = memcg ? m->private : "";
5899 :
5900 : #ifdef CONFIG_MEMCG
5901 : if (memcg)
5902 : cgroup_path(memcg->css.cgroup, m->private, PATH_MAX);
5903 : #endif
5904 : seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path);
5905 : }
5906 :
5907 : seq_printf(m, " node %5d\n", nid);
5908 :
5909 : if (!full)
5910 : seq = min_seq[LRU_GEN_ANON];
5911 : else if (max_seq >= MAX_NR_GENS)
5912 : seq = max_seq - MAX_NR_GENS + 1;
5913 : else
5914 : seq = 0;
5915 :
5916 : for (; seq <= max_seq; seq++) {
5917 : int type, zone;
5918 : int gen = lru_gen_from_seq(seq);
5919 : unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
5920 :
5921 : seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth));
5922 :
5923 : for (type = 0; type < ANON_AND_FILE; type++) {
5924 : unsigned long size = 0;
5925 : char mark = full && seq < min_seq[type] ? 'x' : ' ';
5926 :
5927 : for (zone = 0; zone < MAX_NR_ZONES; zone++)
5928 : size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
5929 :
5930 : seq_printf(m, " %10lu%c", size, mark);
5931 : }
5932 :
5933 : seq_putc(m, '\n');
5934 :
5935 : if (full)
5936 : lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq);
5937 : }
5938 :
5939 : return 0;
5940 : }
5941 :
5942 : static const struct seq_operations lru_gen_seq_ops = {
5943 : .start = lru_gen_seq_start,
5944 : .stop = lru_gen_seq_stop,
5945 : .next = lru_gen_seq_next,
5946 : .show = lru_gen_seq_show,
5947 : };
5948 :
5949 : static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
5950 : bool can_swap, bool force_scan)
5951 : {
5952 : DEFINE_MAX_SEQ(lruvec);
5953 : DEFINE_MIN_SEQ(lruvec);
5954 :
5955 : if (seq < max_seq)
5956 : return 0;
5957 :
5958 : if (seq > max_seq)
5959 : return -EINVAL;
5960 :
5961 : if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq)
5962 : return -ERANGE;
5963 :
5964 : try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan);
5965 :
5966 : return 0;
5967 : }
5968 :
5969 : static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
5970 : int swappiness, unsigned long nr_to_reclaim)
5971 : {
5972 : DEFINE_MAX_SEQ(lruvec);
5973 :
5974 : if (seq + MIN_NR_GENS > max_seq)
5975 : return -EINVAL;
5976 :
5977 : sc->nr_reclaimed = 0;
5978 :
5979 : while (!signal_pending(current)) {
5980 : DEFINE_MIN_SEQ(lruvec);
5981 :
5982 : if (seq < min_seq[!swappiness])
5983 : return 0;
5984 :
5985 : if (sc->nr_reclaimed >= nr_to_reclaim)
5986 : return 0;
5987 :
5988 : if (!evict_folios(lruvec, sc, swappiness))
5989 : return 0;
5990 :
5991 : cond_resched();
5992 : }
5993 :
5994 : return -EINTR;
5995 : }
5996 :
5997 : static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,
5998 : struct scan_control *sc, int swappiness, unsigned long opt)
5999 : {
6000 : struct lruvec *lruvec;
6001 : int err = -EINVAL;
6002 : struct mem_cgroup *memcg = NULL;
6003 :
6004 : if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY))
6005 : return -EINVAL;
6006 :
6007 : if (!mem_cgroup_disabled()) {
6008 : rcu_read_lock();
6009 :
6010 : memcg = mem_cgroup_from_id(memcg_id);
6011 : if (!mem_cgroup_tryget(memcg))
6012 : memcg = NULL;
6013 :
6014 : rcu_read_unlock();
6015 :
6016 : if (!memcg)
6017 : return -EINVAL;
6018 : }
6019 :
6020 : if (memcg_id != mem_cgroup_id(memcg))
6021 : goto done;
6022 :
6023 : lruvec = get_lruvec(memcg, nid);
6024 :
6025 : if (swappiness < 0)
6026 : swappiness = get_swappiness(lruvec, sc);
6027 : else if (swappiness > 200)
6028 : goto done;
6029 :
6030 : switch (cmd) {
6031 : case '+':
6032 : err = run_aging(lruvec, seq, sc, swappiness, opt);
6033 : break;
6034 : case '-':
6035 : err = run_eviction(lruvec, seq, sc, swappiness, opt);
6036 : break;
6037 : }
6038 : done:
6039 : mem_cgroup_put(memcg);
6040 :
6041 : return err;
6042 : }
6043 :
6044 : /* see Documentation/admin-guide/mm/multigen_lru.rst for details */
6045 : static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
6046 : size_t len, loff_t *pos)
6047 : {
6048 : void *buf;
6049 : char *cur, *next;
6050 : unsigned int flags;
6051 : struct blk_plug plug;
6052 : int err = -EINVAL;
6053 : struct scan_control sc = {
6054 : .may_writepage = true,
6055 : .may_unmap = true,
6056 : .may_swap = true,
6057 : .reclaim_idx = MAX_NR_ZONES - 1,
6058 : .gfp_mask = GFP_KERNEL,
6059 : };
6060 :
6061 : buf = kvmalloc(len + 1, GFP_KERNEL);
6062 : if (!buf)
6063 : return -ENOMEM;
6064 :
6065 : if (copy_from_user(buf, src, len)) {
6066 : kvfree(buf);
6067 : return -EFAULT;
6068 : }
6069 :
6070 : set_task_reclaim_state(current, &sc.reclaim_state);
6071 : flags = memalloc_noreclaim_save();
6072 : blk_start_plug(&plug);
6073 : if (!set_mm_walk(NULL, true)) {
6074 : err = -ENOMEM;
6075 : goto done;
6076 : }
6077 :
6078 : next = buf;
6079 : next[len] = '\0';
6080 :
6081 : while ((cur = strsep(&next, ",;\n"))) {
6082 : int n;
6083 : int end;
6084 : char cmd;
6085 : unsigned int memcg_id;
6086 : unsigned int nid;
6087 : unsigned long seq;
6088 : unsigned int swappiness = -1;
6089 : unsigned long opt = -1;
6090 :
6091 : cur = skip_spaces(cur);
6092 : if (!*cur)
6093 : continue;
6094 :
6095 : n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid,
6096 : &seq, &end, &swappiness, &end, &opt, &end);
6097 : if (n < 4 || cur[end]) {
6098 : err = -EINVAL;
6099 : break;
6100 : }
6101 :
6102 : err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt);
6103 : if (err)
6104 : break;
6105 : }
6106 : done:
6107 : clear_mm_walk();
6108 : blk_finish_plug(&plug);
6109 : memalloc_noreclaim_restore(flags);
6110 : set_task_reclaim_state(current, NULL);
6111 :
6112 : kvfree(buf);
6113 :
6114 : return err ? : len;
6115 : }
6116 :
6117 : static int lru_gen_seq_open(struct inode *inode, struct file *file)
6118 : {
6119 : return seq_open(file, &lru_gen_seq_ops);
6120 : }
6121 :
6122 : static const struct file_operations lru_gen_rw_fops = {
6123 : .open = lru_gen_seq_open,
6124 : .read = seq_read,
6125 : .write = lru_gen_seq_write,
6126 : .llseek = seq_lseek,
6127 : .release = seq_release,
6128 : };
6129 :
6130 : static const struct file_operations lru_gen_ro_fops = {
6131 : .open = lru_gen_seq_open,
6132 : .read = seq_read,
6133 : .llseek = seq_lseek,
6134 : .release = seq_release,
6135 : };
6136 :
6137 : /******************************************************************************
6138 : * initialization
6139 : ******************************************************************************/
6140 :
6141 : void lru_gen_init_lruvec(struct lruvec *lruvec)
6142 : {
6143 : int i;
6144 : int gen, type, zone;
6145 : struct lru_gen_folio *lrugen = &lruvec->lrugen;
6146 :
6147 : lrugen->max_seq = MIN_NR_GENS + 1;
6148 : lrugen->enabled = lru_gen_enabled();
6149 :
6150 : for (i = 0; i <= MIN_NR_GENS + 1; i++)
6151 : lrugen->timestamps[i] = jiffies;
6152 :
6153 : for_each_gen_type_zone(gen, type, zone)
6154 : INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
6155 :
6156 : lruvec->mm_state.seq = MIN_NR_GENS;
6157 : }
6158 :
6159 : #ifdef CONFIG_MEMCG
6160 :
6161 : void lru_gen_init_pgdat(struct pglist_data *pgdat)
6162 : {
6163 : int i, j;
6164 :
6165 : spin_lock_init(&pgdat->memcg_lru.lock);
6166 :
6167 : for (i = 0; i < MEMCG_NR_GENS; i++) {
6168 : for (j = 0; j < MEMCG_NR_BINS; j++)
6169 : INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i);
6170 : }
6171 : }
6172 :
6173 : void lru_gen_init_memcg(struct mem_cgroup *memcg)
6174 : {
6175 : INIT_LIST_HEAD(&memcg->mm_list.fifo);
6176 : spin_lock_init(&memcg->mm_list.lock);
6177 : }
6178 :
6179 : void lru_gen_exit_memcg(struct mem_cgroup *memcg)
6180 : {
6181 : int i;
6182 : int nid;
6183 :
6184 : VM_WARN_ON_ONCE(!list_empty(&memcg->mm_list.fifo));
6185 :
6186 : for_each_node(nid) {
6187 : struct lruvec *lruvec = get_lruvec(memcg, nid);
6188 :
6189 : VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
6190 : sizeof(lruvec->lrugen.nr_pages)));
6191 :
6192 : lruvec->lrugen.list.next = LIST_POISON1;
6193 :
6194 : for (i = 0; i < NR_BLOOM_FILTERS; i++) {
6195 : bitmap_free(lruvec->mm_state.filters[i]);
6196 : lruvec->mm_state.filters[i] = NULL;
6197 : }
6198 : }
6199 : }
6200 :
6201 : #endif /* CONFIG_MEMCG */
6202 :
6203 : static int __init init_lru_gen(void)
6204 : {
6205 : BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS);
6206 : BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS);
6207 :
6208 : if (sysfs_create_group(mm_kobj, &lru_gen_attr_group))
6209 : pr_err("lru_gen: failed to create sysfs group\n");
6210 :
6211 : debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops);
6212 : debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops);
6213 :
6214 : return 0;
6215 : };
6216 : late_initcall(init_lru_gen);
6217 :
6218 : #else /* !CONFIG_LRU_GEN */
6219 :
6220 : static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
6221 : {
6222 : }
6223 :
6224 : static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
6225 : {
6226 : }
6227 :
6228 : static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc)
6229 : {
6230 : }
6231 :
6232 : #endif /* CONFIG_LRU_GEN */
6233 :
6234 0 : static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
6235 : {
6236 : unsigned long nr[NR_LRU_LISTS];
6237 : unsigned long targets[NR_LRU_LISTS];
6238 : unsigned long nr_to_scan;
6239 : enum lru_list lru;
6240 0 : unsigned long nr_reclaimed = 0;
6241 0 : unsigned long nr_to_reclaim = sc->nr_to_reclaim;
6242 : bool proportional_reclaim;
6243 : struct blk_plug plug;
6244 :
6245 : if (lru_gen_enabled() && !global_reclaim(sc)) {
6246 : lru_gen_shrink_lruvec(lruvec, sc);
6247 : return;
6248 : }
6249 :
6250 0 : get_scan_count(lruvec, sc, nr);
6251 :
6252 : /* Record the original scan target for proportional adjustments later */
6253 0 : memcpy(targets, nr, sizeof(nr));
6254 :
6255 : /*
6256 : * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
6257 : * event that can occur when there is little memory pressure e.g.
6258 : * multiple streaming readers/writers. Hence, we do not abort scanning
6259 : * when the requested number of pages are reclaimed when scanning at
6260 : * DEF_PRIORITY on the assumption that the fact we are direct
6261 : * reclaiming implies that kswapd is not keeping up and it is best to
6262 : * do a batch of work at once. For memcg reclaim one check is made to
6263 : * abort proportional reclaim if either the file or anon lru has already
6264 : * dropped to zero at the first pass.
6265 : */
6266 0 : proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
6267 0 : sc->priority == DEF_PRIORITY);
6268 :
6269 0 : blk_start_plug(&plug);
6270 0 : while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
6271 0 : nr[LRU_INACTIVE_FILE]) {
6272 : unsigned long nr_anon, nr_file, percentage;
6273 : unsigned long nr_scanned;
6274 :
6275 0 : for_each_evictable_lru(lru) {
6276 0 : if (nr[lru]) {
6277 0 : nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
6278 0 : nr[lru] -= nr_to_scan;
6279 :
6280 0 : nr_reclaimed += shrink_list(lru, nr_to_scan,
6281 : lruvec, sc);
6282 : }
6283 : }
6284 :
6285 0 : cond_resched();
6286 :
6287 0 : if (nr_reclaimed < nr_to_reclaim || proportional_reclaim)
6288 0 : continue;
6289 :
6290 : /*
6291 : * For kswapd and memcg, reclaim at least the number of pages
6292 : * requested. Ensure that the anon and file LRUs are scanned
6293 : * proportionally what was requested by get_scan_count(). We
6294 : * stop reclaiming one LRU and reduce the amount scanning
6295 : * proportional to the original scan target.
6296 : */
6297 0 : nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
6298 0 : nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
6299 :
6300 : /*
6301 : * It's just vindictive to attack the larger once the smaller
6302 : * has gone to zero. And given the way we stop scanning the
6303 : * smaller below, this makes sure that we only make one nudge
6304 : * towards proportionality once we've got nr_to_reclaim.
6305 : */
6306 0 : if (!nr_file || !nr_anon)
6307 : break;
6308 :
6309 0 : if (nr_file > nr_anon) {
6310 0 : unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
6311 0 : targets[LRU_ACTIVE_ANON] + 1;
6312 0 : lru = LRU_BASE;
6313 0 : percentage = nr_anon * 100 / scan_target;
6314 : } else {
6315 0 : unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
6316 0 : targets[LRU_ACTIVE_FILE] + 1;
6317 0 : lru = LRU_FILE;
6318 0 : percentage = nr_file * 100 / scan_target;
6319 : }
6320 :
6321 : /* Stop scanning the smaller of the LRU */
6322 0 : nr[lru] = 0;
6323 0 : nr[lru + LRU_ACTIVE] = 0;
6324 :
6325 : /*
6326 : * Recalculate the other LRU scan count based on its original
6327 : * scan target and the percentage scanning already complete
6328 : */
6329 0 : lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
6330 0 : nr_scanned = targets[lru] - nr[lru];
6331 0 : nr[lru] = targets[lru] * (100 - percentage) / 100;
6332 0 : nr[lru] -= min(nr[lru], nr_scanned);
6333 :
6334 0 : lru += LRU_ACTIVE;
6335 0 : nr_scanned = targets[lru] - nr[lru];
6336 0 : nr[lru] = targets[lru] * (100 - percentage) / 100;
6337 0 : nr[lru] -= min(nr[lru], nr_scanned);
6338 : }
6339 0 : blk_finish_plug(&plug);
6340 0 : sc->nr_reclaimed += nr_reclaimed;
6341 :
6342 : /*
6343 : * Even if we did not try to evict anon pages at all, we want to
6344 : * rebalance the anon lru active/inactive ratio.
6345 : */
6346 0 : if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) &&
6347 0 : inactive_is_low(lruvec, LRU_INACTIVE_ANON))
6348 0 : shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
6349 : sc, LRU_ACTIVE_ANON);
6350 : }
6351 :
6352 : /* Use reclaim/compaction for costly allocs or under memory pressure */
6353 : static bool in_reclaim_compaction(struct scan_control *sc)
6354 : {
6355 0 : if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
6356 0 : (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
6357 0 : sc->priority < DEF_PRIORITY - 2))
6358 : return true;
6359 :
6360 : return false;
6361 : }
6362 :
6363 : /*
6364 : * Reclaim/compaction is used for high-order allocation requests. It reclaims
6365 : * order-0 pages before compacting the zone. should_continue_reclaim() returns
6366 : * true if more pages should be reclaimed such that when the page allocator
6367 : * calls try_to_compact_pages() that it will have enough free pages to succeed.
6368 : * It will give up earlier than that if there is difficulty reclaiming pages.
6369 : */
6370 0 : static inline bool should_continue_reclaim(struct pglist_data *pgdat,
6371 : unsigned long nr_reclaimed,
6372 : struct scan_control *sc)
6373 : {
6374 : unsigned long pages_for_compaction;
6375 : unsigned long inactive_lru_pages;
6376 : int z;
6377 :
6378 : /* If not in reclaim/compaction mode, stop */
6379 0 : if (!in_reclaim_compaction(sc))
6380 : return false;
6381 :
6382 : /*
6383 : * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
6384 : * number of pages that were scanned. This will return to the caller
6385 : * with the risk reclaim/compaction and the resulting allocation attempt
6386 : * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
6387 : * allocations through requiring that the full LRU list has been scanned
6388 : * first, by assuming that zero delta of sc->nr_scanned means full LRU
6389 : * scan, but that approximation was wrong, and there were corner cases
6390 : * where always a non-zero amount of pages were scanned.
6391 : */
6392 0 : if (!nr_reclaimed)
6393 : return false;
6394 :
6395 : /* If compaction would go ahead or the allocation would succeed, stop */
6396 0 : for (z = 0; z <= sc->reclaim_idx; z++) {
6397 0 : struct zone *zone = &pgdat->node_zones[z];
6398 0 : if (!managed_zone(zone))
6399 0 : continue;
6400 :
6401 0 : switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
6402 : case COMPACT_SUCCESS:
6403 : case COMPACT_CONTINUE:
6404 : return false;
6405 : default:
6406 : /* check next zone */
6407 : ;
6408 : }
6409 : }
6410 :
6411 : /*
6412 : * If we have not reclaimed enough pages for compaction and the
6413 : * inactive lists are large enough, continue reclaiming
6414 : */
6415 0 : pages_for_compaction = compact_gap(sc->order);
6416 0 : inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
6417 0 : if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))
6418 0 : inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
6419 :
6420 0 : return inactive_lru_pages > pages_for_compaction;
6421 : }
6422 :
6423 0 : static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
6424 : {
6425 0 : struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
6426 : struct mem_cgroup *memcg;
6427 :
6428 0 : memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
6429 : do {
6430 0 : struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
6431 : unsigned long reclaimed;
6432 : unsigned long scanned;
6433 :
6434 : /*
6435 : * This loop can become CPU-bound when target memcgs
6436 : * aren't eligible for reclaim - either because they
6437 : * don't have any reclaimable pages, or because their
6438 : * memory is explicitly protected. Avoid soft lockups.
6439 : */
6440 0 : cond_resched();
6441 :
6442 0 : mem_cgroup_calculate_protection(target_memcg, memcg);
6443 :
6444 0 : if (mem_cgroup_below_min(target_memcg, memcg)) {
6445 : /*
6446 : * Hard protection.
6447 : * If there is no reclaimable memory, OOM.
6448 : */
6449 : continue;
6450 0 : } else if (mem_cgroup_below_low(target_memcg, memcg)) {
6451 : /*
6452 : * Soft protection.
6453 : * Respect the protection only as long as
6454 : * there is an unprotected supply
6455 : * of reclaimable memory from other cgroups.
6456 : */
6457 : if (!sc->memcg_low_reclaim) {
6458 : sc->memcg_low_skipped = 1;
6459 : continue;
6460 : }
6461 : memcg_memory_event(memcg, MEMCG_LOW);
6462 : }
6463 :
6464 0 : reclaimed = sc->nr_reclaimed;
6465 0 : scanned = sc->nr_scanned;
6466 :
6467 0 : shrink_lruvec(lruvec, sc);
6468 :
6469 0 : shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
6470 0 : sc->priority);
6471 :
6472 : /* Record the group's reclaim efficiency */
6473 : if (!sc->proactive)
6474 : vmpressure(sc->gfp_mask, memcg, false,
6475 : sc->nr_scanned - scanned,
6476 : sc->nr_reclaimed - reclaimed);
6477 :
6478 0 : } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
6479 0 : }
6480 :
6481 0 : static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
6482 : {
6483 : unsigned long nr_reclaimed, nr_scanned, nr_node_reclaimed;
6484 : struct lruvec *target_lruvec;
6485 0 : bool reclaimable = false;
6486 :
6487 : if (lru_gen_enabled() && global_reclaim(sc)) {
6488 : lru_gen_shrink_node(pgdat, sc);
6489 : return;
6490 : }
6491 :
6492 0 : target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
6493 :
6494 : again:
6495 0 : memset(&sc->nr, 0, sizeof(sc->nr));
6496 :
6497 0 : nr_reclaimed = sc->nr_reclaimed;
6498 0 : nr_scanned = sc->nr_scanned;
6499 :
6500 0 : prepare_scan_count(pgdat, sc);
6501 :
6502 0 : shrink_node_memcgs(pgdat, sc);
6503 :
6504 0 : flush_reclaim_state(sc);
6505 :
6506 0 : nr_node_reclaimed = sc->nr_reclaimed - nr_reclaimed;
6507 :
6508 : /* Record the subtree's reclaim efficiency */
6509 : if (!sc->proactive)
6510 : vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
6511 : sc->nr_scanned - nr_scanned, nr_node_reclaimed);
6512 :
6513 0 : if (nr_node_reclaimed)
6514 0 : reclaimable = true;
6515 :
6516 0 : if (current_is_kswapd()) {
6517 : /*
6518 : * If reclaim is isolating dirty pages under writeback,
6519 : * it implies that the long-lived page allocation rate
6520 : * is exceeding the page laundering rate. Either the
6521 : * global limits are not being effective at throttling
6522 : * processes due to the page distribution throughout
6523 : * zones or there is heavy usage of a slow backing
6524 : * device. The only option is to throttle from reclaim
6525 : * context which is not ideal as there is no guarantee
6526 : * the dirtying process is throttled in the same way
6527 : * balance_dirty_pages() manages.
6528 : *
6529 : * Once a node is flagged PGDAT_WRITEBACK, kswapd will
6530 : * count the number of pages under pages flagged for
6531 : * immediate reclaim and stall if any are encountered
6532 : * in the nr_immediate check below.
6533 : */
6534 0 : if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
6535 0 : set_bit(PGDAT_WRITEBACK, &pgdat->flags);
6536 :
6537 : /* Allow kswapd to start writing pages during reclaim.*/
6538 0 : if (sc->nr.unqueued_dirty == sc->nr.file_taken)
6539 0 : set_bit(PGDAT_DIRTY, &pgdat->flags);
6540 :
6541 : /*
6542 : * If kswapd scans pages marked for immediate
6543 : * reclaim and under writeback (nr_immediate), it
6544 : * implies that pages are cycling through the LRU
6545 : * faster than they are written so forcibly stall
6546 : * until some pages complete writeback.
6547 : */
6548 0 : if (sc->nr.immediate)
6549 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
6550 : }
6551 :
6552 : /*
6553 : * Tag a node/memcg as congested if all the dirty pages were marked
6554 : * for writeback and immediate reclaim (counted in nr.congested).
6555 : *
6556 : * Legacy memcg will stall in page writeback so avoid forcibly
6557 : * stalling in reclaim_throttle().
6558 : */
6559 0 : if ((current_is_kswapd() ||
6560 0 : (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
6561 0 : sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
6562 0 : set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
6563 :
6564 : /*
6565 : * Stall direct reclaim for IO completions if the lruvec is
6566 : * node is congested. Allow kswapd to continue until it
6567 : * starts encountering unqueued dirty pages or cycling through
6568 : * the LRU too quickly.
6569 : */
6570 0 : if (!current_is_kswapd() && current_may_throttle() &&
6571 0 : !sc->hibernation_mode &&
6572 0 : test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
6573 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED);
6574 :
6575 0 : if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc))
6576 : goto again;
6577 :
6578 : /*
6579 : * Kswapd gives up on balancing particular nodes after too
6580 : * many failures to reclaim anything from them and goes to
6581 : * sleep. On reclaim progress, reset the failure counter. A
6582 : * successful direct reclaim run will revive a dormant kswapd.
6583 : */
6584 0 : if (reclaimable)
6585 0 : pgdat->kswapd_failures = 0;
6586 : }
6587 :
6588 : /*
6589 : * Returns true if compaction should go ahead for a costly-order request, or
6590 : * the allocation would already succeed without compaction. Return false if we
6591 : * should reclaim first.
6592 : */
6593 0 : static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
6594 : {
6595 : unsigned long watermark;
6596 : enum compact_result suitable;
6597 :
6598 0 : suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
6599 0 : if (suitable == COMPACT_SUCCESS)
6600 : /* Allocation should succeed already. Don't reclaim. */
6601 : return true;
6602 0 : if (suitable == COMPACT_SKIPPED)
6603 : /* Compaction cannot yet proceed. Do reclaim. */
6604 : return false;
6605 :
6606 : /*
6607 : * Compaction is already possible, but it takes time to run and there
6608 : * are potentially other callers using the pages just freed. So proceed
6609 : * with reclaim to make a buffer of free pages available to give
6610 : * compaction a reasonable chance of completing and allocating the page.
6611 : * Note that we won't actually reclaim the whole buffer in one attempt
6612 : * as the target watermark in should_continue_reclaim() is lower. But if
6613 : * we are already above the high+gap watermark, don't reclaim at all.
6614 : */
6615 0 : watermark = high_wmark_pages(zone) + compact_gap(sc->order);
6616 :
6617 0 : return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
6618 : }
6619 :
6620 0 : static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc)
6621 : {
6622 : /*
6623 : * If reclaim is making progress greater than 12% efficiency then
6624 : * wake all the NOPROGRESS throttled tasks.
6625 : */
6626 0 : if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) {
6627 : wait_queue_head_t *wqh;
6628 :
6629 0 : wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS];
6630 0 : if (waitqueue_active(wqh))
6631 0 : wake_up(wqh);
6632 :
6633 : return;
6634 : }
6635 :
6636 : /*
6637 : * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will
6638 : * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages
6639 : * under writeback and marked for immediate reclaim at the tail of the
6640 : * LRU.
6641 : */
6642 0 : if (current_is_kswapd() || cgroup_reclaim(sc))
6643 : return;
6644 :
6645 : /* Throttle if making no progress at high prioities. */
6646 0 : if (sc->priority == 1 && !sc->nr_reclaimed)
6647 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS);
6648 : }
6649 :
6650 : /*
6651 : * This is the direct reclaim path, for page-allocating processes. We only
6652 : * try to reclaim pages from zones which will satisfy the caller's allocation
6653 : * request.
6654 : *
6655 : * If a zone is deemed to be full of pinned pages then just give it a light
6656 : * scan then give up on it.
6657 : */
6658 0 : static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
6659 : {
6660 : struct zoneref *z;
6661 : struct zone *zone;
6662 : unsigned long nr_soft_reclaimed;
6663 : unsigned long nr_soft_scanned;
6664 : gfp_t orig_mask;
6665 0 : pg_data_t *last_pgdat = NULL;
6666 0 : pg_data_t *first_pgdat = NULL;
6667 :
6668 : /*
6669 : * If the number of buffer_heads in the machine exceeds the maximum
6670 : * allowed level, force direct reclaim to scan the highmem zone as
6671 : * highmem pages could be pinning lowmem pages storing buffer_heads
6672 : */
6673 0 : orig_mask = sc->gfp_mask;
6674 0 : if (buffer_heads_over_limit) {
6675 0 : sc->gfp_mask |= __GFP_HIGHMEM;
6676 0 : sc->reclaim_idx = gfp_zone(sc->gfp_mask);
6677 : }
6678 :
6679 0 : for_each_zone_zonelist_nodemask(zone, z, zonelist,
6680 : sc->reclaim_idx, sc->nodemask) {
6681 : /*
6682 : * Take care memory controller reclaiming has small influence
6683 : * to global LRU.
6684 : */
6685 0 : if (!cgroup_reclaim(sc)) {
6686 0 : if (!cpuset_zone_allowed(zone,
6687 : GFP_KERNEL | __GFP_HARDWALL))
6688 : continue;
6689 :
6690 : /*
6691 : * If we already have plenty of memory free for
6692 : * compaction in this zone, don't free any more.
6693 : * Even though compaction is invoked for any
6694 : * non-zero order, only frequent costly order
6695 : * reclamation is disruptive enough to become a
6696 : * noticeable problem, like transparent huge
6697 : * page allocations.
6698 : */
6699 0 : if (IS_ENABLED(CONFIG_COMPACTION) &&
6700 0 : sc->order > PAGE_ALLOC_COSTLY_ORDER &&
6701 0 : compaction_ready(zone, sc)) {
6702 0 : sc->compaction_ready = true;
6703 0 : continue;
6704 : }
6705 :
6706 : /*
6707 : * Shrink each node in the zonelist once. If the
6708 : * zonelist is ordered by zone (not the default) then a
6709 : * node may be shrunk multiple times but in that case
6710 : * the user prefers lower zones being preserved.
6711 : */
6712 0 : if (zone->zone_pgdat == last_pgdat)
6713 0 : continue;
6714 :
6715 : /*
6716 : * This steals pages from memory cgroups over softlimit
6717 : * and returns the number of reclaimed pages and
6718 : * scanned pages. This works for global memory pressure
6719 : * and balancing, not for a memcg's limit.
6720 : */
6721 0 : nr_soft_scanned = 0;
6722 0 : nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
6723 0 : sc->order, sc->gfp_mask,
6724 : &nr_soft_scanned);
6725 : sc->nr_reclaimed += nr_soft_reclaimed;
6726 : sc->nr_scanned += nr_soft_scanned;
6727 : /* need some check for avoid more shrink_zone() */
6728 : }
6729 :
6730 0 : if (!first_pgdat)
6731 0 : first_pgdat = zone->zone_pgdat;
6732 :
6733 : /* See comment about same check for global reclaim above */
6734 : if (zone->zone_pgdat == last_pgdat)
6735 : continue;
6736 0 : last_pgdat = zone->zone_pgdat;
6737 0 : shrink_node(zone->zone_pgdat, sc);
6738 : }
6739 :
6740 0 : if (first_pgdat)
6741 0 : consider_reclaim_throttle(first_pgdat, sc);
6742 :
6743 : /*
6744 : * Restore to original mask to avoid the impact on the caller if we
6745 : * promoted it to __GFP_HIGHMEM.
6746 : */
6747 0 : sc->gfp_mask = orig_mask;
6748 0 : }
6749 :
6750 : static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)
6751 : {
6752 : struct lruvec *target_lruvec;
6753 : unsigned long refaults;
6754 :
6755 : if (lru_gen_enabled())
6756 : return;
6757 :
6758 0 : target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
6759 0 : refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
6760 0 : target_lruvec->refaults[WORKINGSET_ANON] = refaults;
6761 0 : refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
6762 0 : target_lruvec->refaults[WORKINGSET_FILE] = refaults;
6763 : }
6764 :
6765 : /*
6766 : * This is the main entry point to direct page reclaim.
6767 : *
6768 : * If a full scan of the inactive list fails to free enough memory then we
6769 : * are "out of memory" and something needs to be killed.
6770 : *
6771 : * If the caller is !__GFP_FS then the probability of a failure is reasonably
6772 : * high - the zone may be full of dirty or under-writeback pages, which this
6773 : * caller can't do much about. We kick the writeback threads and take explicit
6774 : * naps in the hope that some of these pages can be written. But if the
6775 : * allocating task holds filesystem locks which prevent writeout this might not
6776 : * work, and the allocation attempt will fail.
6777 : *
6778 : * returns: 0, if no pages reclaimed
6779 : * else, the number of pages reclaimed
6780 : */
6781 0 : static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
6782 : struct scan_control *sc)
6783 : {
6784 0 : int initial_priority = sc->priority;
6785 : pg_data_t *last_pgdat;
6786 : struct zoneref *z;
6787 : struct zone *zone;
6788 : retry:
6789 : delayacct_freepages_start();
6790 :
6791 0 : if (!cgroup_reclaim(sc))
6792 0 : __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
6793 :
6794 : do {
6795 : if (!sc->proactive)
6796 : vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
6797 : sc->priority);
6798 0 : sc->nr_scanned = 0;
6799 0 : shrink_zones(zonelist, sc);
6800 :
6801 0 : if (sc->nr_reclaimed >= sc->nr_to_reclaim)
6802 : break;
6803 :
6804 0 : if (sc->compaction_ready)
6805 : break;
6806 :
6807 : /*
6808 : * If we're getting trouble reclaiming, start doing
6809 : * writepage even in laptop mode.
6810 : */
6811 0 : if (sc->priority < DEF_PRIORITY - 2)
6812 0 : sc->may_writepage = 1;
6813 0 : } while (--sc->priority >= 0);
6814 :
6815 0 : last_pgdat = NULL;
6816 0 : for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
6817 : sc->nodemask) {
6818 0 : if (zone->zone_pgdat == last_pgdat)
6819 0 : continue;
6820 0 : last_pgdat = zone->zone_pgdat;
6821 :
6822 0 : snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
6823 :
6824 0 : if (cgroup_reclaim(sc)) {
6825 : struct lruvec *lruvec;
6826 :
6827 : lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
6828 : zone->zone_pgdat);
6829 : clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
6830 : }
6831 : }
6832 :
6833 : delayacct_freepages_end();
6834 :
6835 0 : if (sc->nr_reclaimed)
6836 : return sc->nr_reclaimed;
6837 :
6838 : /* Aborted reclaim to try compaction? don't OOM, then */
6839 0 : if (sc->compaction_ready)
6840 : return 1;
6841 :
6842 : /*
6843 : * We make inactive:active ratio decisions based on the node's
6844 : * composition of memory, but a restrictive reclaim_idx or a
6845 : * memory.low cgroup setting can exempt large amounts of
6846 : * memory from reclaim. Neither of which are very common, so
6847 : * instead of doing costly eligibility calculations of the
6848 : * entire cgroup subtree up front, we assume the estimates are
6849 : * good, and retry with forcible deactivation if that fails.
6850 : */
6851 0 : if (sc->skipped_deactivate) {
6852 0 : sc->priority = initial_priority;
6853 0 : sc->force_deactivate = 1;
6854 0 : sc->skipped_deactivate = 0;
6855 0 : goto retry;
6856 : }
6857 :
6858 : /* Untapped cgroup reserves? Don't OOM, retry. */
6859 0 : if (sc->memcg_low_skipped) {
6860 0 : sc->priority = initial_priority;
6861 0 : sc->force_deactivate = 0;
6862 0 : sc->memcg_low_reclaim = 1;
6863 0 : sc->memcg_low_skipped = 0;
6864 0 : goto retry;
6865 : }
6866 :
6867 : return 0;
6868 : }
6869 :
6870 0 : static bool allow_direct_reclaim(pg_data_t *pgdat)
6871 : {
6872 : struct zone *zone;
6873 0 : unsigned long pfmemalloc_reserve = 0;
6874 0 : unsigned long free_pages = 0;
6875 : int i;
6876 : bool wmark_ok;
6877 :
6878 0 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
6879 : return true;
6880 :
6881 0 : for (i = 0; i <= ZONE_NORMAL; i++) {
6882 0 : zone = &pgdat->node_zones[i];
6883 0 : if (!managed_zone(zone))
6884 0 : continue;
6885 :
6886 0 : if (!zone_reclaimable_pages(zone))
6887 0 : continue;
6888 :
6889 0 : pfmemalloc_reserve += min_wmark_pages(zone);
6890 0 : free_pages += zone_page_state(zone, NR_FREE_PAGES);
6891 : }
6892 :
6893 : /* If there are no reserves (unexpected config) then do not throttle */
6894 0 : if (!pfmemalloc_reserve)
6895 : return true;
6896 :
6897 0 : wmark_ok = free_pages > pfmemalloc_reserve / 2;
6898 :
6899 : /* kswapd must be awake if processes are being throttled */
6900 0 : if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
6901 0 : if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
6902 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
6903 :
6904 0 : wake_up_interruptible(&pgdat->kswapd_wait);
6905 : }
6906 :
6907 : return wmark_ok;
6908 : }
6909 :
6910 : /*
6911 : * Throttle direct reclaimers if backing storage is backed by the network
6912 : * and the PFMEMALLOC reserve for the preferred node is getting dangerously
6913 : * depleted. kswapd will continue to make progress and wake the processes
6914 : * when the low watermark is reached.
6915 : *
6916 : * Returns true if a fatal signal was delivered during throttling. If this
6917 : * happens, the page allocator should not consider triggering the OOM killer.
6918 : */
6919 0 : static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
6920 : nodemask_t *nodemask)
6921 : {
6922 : struct zoneref *z;
6923 : struct zone *zone;
6924 0 : pg_data_t *pgdat = NULL;
6925 :
6926 : /*
6927 : * Kernel threads should not be throttled as they may be indirectly
6928 : * responsible for cleaning pages necessary for reclaim to make forward
6929 : * progress. kjournald for example may enter direct reclaim while
6930 : * committing a transaction where throttling it could forcing other
6931 : * processes to block on log_wait_commit().
6932 : */
6933 0 : if (current->flags & PF_KTHREAD)
6934 : goto out;
6935 :
6936 : /*
6937 : * If a fatal signal is pending, this process should not throttle.
6938 : * It should return quickly so it can exit and free its memory
6939 : */
6940 0 : if (fatal_signal_pending(current))
6941 : goto out;
6942 :
6943 : /*
6944 : * Check if the pfmemalloc reserves are ok by finding the first node
6945 : * with a usable ZONE_NORMAL or lower zone. The expectation is that
6946 : * GFP_KERNEL will be required for allocating network buffers when
6947 : * swapping over the network so ZONE_HIGHMEM is unusable.
6948 : *
6949 : * Throttling is based on the first usable node and throttled processes
6950 : * wait on a queue until kswapd makes progress and wakes them. There
6951 : * is an affinity then between processes waking up and where reclaim
6952 : * progress has been made assuming the process wakes on the same node.
6953 : * More importantly, processes running on remote nodes will not compete
6954 : * for remote pfmemalloc reserves and processes on different nodes
6955 : * should make reasonable progress.
6956 : */
6957 0 : for_each_zone_zonelist_nodemask(zone, z, zonelist,
6958 : gfp_zone(gfp_mask), nodemask) {
6959 0 : if (zone_idx(zone) > ZONE_NORMAL)
6960 0 : continue;
6961 :
6962 : /* Throttle based on the first usable node */
6963 0 : pgdat = zone->zone_pgdat;
6964 0 : if (allow_direct_reclaim(pgdat))
6965 : goto out;
6966 : break;
6967 : }
6968 :
6969 : /* If no zone was usable by the allocation flags then do not throttle */
6970 0 : if (!pgdat)
6971 : goto out;
6972 :
6973 : /* Account for the throttling */
6974 0 : count_vm_event(PGSCAN_DIRECT_THROTTLE);
6975 :
6976 : /*
6977 : * If the caller cannot enter the filesystem, it's possible that it
6978 : * is due to the caller holding an FS lock or performing a journal
6979 : * transaction in the case of a filesystem like ext[3|4]. In this case,
6980 : * it is not safe to block on pfmemalloc_wait as kswapd could be
6981 : * blocked waiting on the same lock. Instead, throttle for up to a
6982 : * second before continuing.
6983 : */
6984 0 : if (!(gfp_mask & __GFP_FS))
6985 0 : wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
6986 : allow_direct_reclaim(pgdat), HZ);
6987 : else
6988 : /* Throttle until kswapd wakes the process */
6989 0 : wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
6990 : allow_direct_reclaim(pgdat));
6991 :
6992 0 : if (fatal_signal_pending(current))
6993 : return true;
6994 :
6995 : out:
6996 : return false;
6997 : }
6998 :
6999 0 : unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
7000 : gfp_t gfp_mask, nodemask_t *nodemask)
7001 : {
7002 : unsigned long nr_reclaimed;
7003 0 : struct scan_control sc = {
7004 : .nr_to_reclaim = SWAP_CLUSTER_MAX,
7005 0 : .gfp_mask = current_gfp_context(gfp_mask),
7006 0 : .reclaim_idx = gfp_zone(gfp_mask),
7007 : .order = order,
7008 : .nodemask = nodemask,
7009 : .priority = DEF_PRIORITY,
7010 0 : .may_writepage = !laptop_mode,
7011 : .may_unmap = 1,
7012 : .may_swap = 1,
7013 : };
7014 :
7015 : /*
7016 : * scan_control uses s8 fields for order, priority, and reclaim_idx.
7017 : * Confirm they are large enough for max values.
7018 : */
7019 : BUILD_BUG_ON(MAX_ORDER >= S8_MAX);
7020 : BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
7021 : BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
7022 :
7023 : /*
7024 : * Do not enter reclaim if fatal signal was delivered while throttled.
7025 : * 1 is returned so that the page allocator does not OOM kill at this
7026 : * point.
7027 : */
7028 0 : if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
7029 : return 1;
7030 :
7031 0 : set_task_reclaim_state(current, &sc.reclaim_state);
7032 0 : trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
7033 :
7034 0 : nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
7035 :
7036 0 : trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
7037 0 : set_task_reclaim_state(current, NULL);
7038 :
7039 0 : return nr_reclaimed;
7040 : }
7041 :
7042 : #ifdef CONFIG_MEMCG
7043 :
7044 : /* Only used by soft limit reclaim. Do not reuse for anything else. */
7045 : unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
7046 : gfp_t gfp_mask, bool noswap,
7047 : pg_data_t *pgdat,
7048 : unsigned long *nr_scanned)
7049 : {
7050 : struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
7051 : struct scan_control sc = {
7052 : .nr_to_reclaim = SWAP_CLUSTER_MAX,
7053 : .target_mem_cgroup = memcg,
7054 : .may_writepage = !laptop_mode,
7055 : .may_unmap = 1,
7056 : .reclaim_idx = MAX_NR_ZONES - 1,
7057 : .may_swap = !noswap,
7058 : };
7059 :
7060 : WARN_ON_ONCE(!current->reclaim_state);
7061 :
7062 : sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
7063 : (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
7064 :
7065 : trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
7066 : sc.gfp_mask);
7067 :
7068 : /*
7069 : * NOTE: Although we can get the priority field, using it
7070 : * here is not a good idea, since it limits the pages we can scan.
7071 : * if we don't reclaim here, the shrink_node from balance_pgdat
7072 : * will pick up pages from other mem cgroup's as well. We hack
7073 : * the priority and make it zero.
7074 : */
7075 : shrink_lruvec(lruvec, &sc);
7076 :
7077 : trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
7078 :
7079 : *nr_scanned = sc.nr_scanned;
7080 :
7081 : return sc.nr_reclaimed;
7082 : }
7083 :
7084 : unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
7085 : unsigned long nr_pages,
7086 : gfp_t gfp_mask,
7087 : unsigned int reclaim_options)
7088 : {
7089 : unsigned long nr_reclaimed;
7090 : unsigned int noreclaim_flag;
7091 : struct scan_control sc = {
7092 : .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
7093 : .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
7094 : (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
7095 : .reclaim_idx = MAX_NR_ZONES - 1,
7096 : .target_mem_cgroup = memcg,
7097 : .priority = DEF_PRIORITY,
7098 : .may_writepage = !laptop_mode,
7099 : .may_unmap = 1,
7100 : .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP),
7101 : .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE),
7102 : };
7103 : /*
7104 : * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
7105 : * equal pressure on all the nodes. This is based on the assumption that
7106 : * the reclaim does not bail out early.
7107 : */
7108 : struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
7109 :
7110 : set_task_reclaim_state(current, &sc.reclaim_state);
7111 : trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
7112 : noreclaim_flag = memalloc_noreclaim_save();
7113 :
7114 : nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
7115 :
7116 : memalloc_noreclaim_restore(noreclaim_flag);
7117 : trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
7118 : set_task_reclaim_state(current, NULL);
7119 :
7120 : return nr_reclaimed;
7121 : }
7122 : #endif
7123 :
7124 0 : static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc)
7125 : {
7126 : struct mem_cgroup *memcg;
7127 : struct lruvec *lruvec;
7128 :
7129 : if (lru_gen_enabled()) {
7130 : lru_gen_age_node(pgdat, sc);
7131 : return;
7132 : }
7133 :
7134 0 : if (!can_age_anon_pages(pgdat, sc))
7135 : return;
7136 :
7137 0 : lruvec = mem_cgroup_lruvec(NULL, pgdat);
7138 0 : if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
7139 : return;
7140 :
7141 0 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
7142 : do {
7143 0 : lruvec = mem_cgroup_lruvec(memcg, pgdat);
7144 0 : shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
7145 : sc, LRU_ACTIVE_ANON);
7146 0 : memcg = mem_cgroup_iter(NULL, memcg, NULL);
7147 : } while (memcg);
7148 : }
7149 :
7150 : static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
7151 : {
7152 : int i;
7153 : struct zone *zone;
7154 :
7155 : /*
7156 : * Check for watermark boosts top-down as the higher zones
7157 : * are more likely to be boosted. Both watermarks and boosts
7158 : * should not be checked at the same time as reclaim would
7159 : * start prematurely when there is no boosting and a lower
7160 : * zone is balanced.
7161 : */
7162 0 : for (i = highest_zoneidx; i >= 0; i--) {
7163 0 : zone = pgdat->node_zones + i;
7164 0 : if (!managed_zone(zone))
7165 0 : continue;
7166 :
7167 0 : if (zone->watermark_boost)
7168 : return true;
7169 : }
7170 :
7171 : return false;
7172 : }
7173 :
7174 : /*
7175 : * Returns true if there is an eligible zone balanced for the request order
7176 : * and highest_zoneidx
7177 : */
7178 2 : static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
7179 : {
7180 : int i;
7181 2 : unsigned long mark = -1;
7182 : struct zone *zone;
7183 :
7184 : /*
7185 : * Check watermarks bottom-up as lower zones are more likely to
7186 : * meet watermarks.
7187 : */
7188 2 : for (i = 0; i <= highest_zoneidx; i++) {
7189 2 : zone = pgdat->node_zones + i;
7190 :
7191 2 : if (!managed_zone(zone))
7192 0 : continue;
7193 :
7194 : if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
7195 : mark = wmark_pages(zone, WMARK_PROMO);
7196 : else
7197 2 : mark = high_wmark_pages(zone);
7198 2 : if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
7199 : return true;
7200 : }
7201 :
7202 : /*
7203 : * If a node has no managed zone within highest_zoneidx, it does not
7204 : * need balancing by definition. This can happen if a zone-restricted
7205 : * allocation tries to wake a remote kswapd.
7206 : */
7207 0 : if (mark == -1)
7208 : return true;
7209 :
7210 0 : return false;
7211 : }
7212 :
7213 : /* Clear pgdat state for congested, dirty or under writeback. */
7214 : static void clear_pgdat_congested(pg_data_t *pgdat)
7215 : {
7216 2 : struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
7217 :
7218 4 : clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
7219 4 : clear_bit(PGDAT_DIRTY, &pgdat->flags);
7220 4 : clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
7221 : }
7222 :
7223 : /*
7224 : * Prepare kswapd for sleeping. This verifies that there are no processes
7225 : * waiting in throttle_direct_reclaim() and that watermarks have been met.
7226 : *
7227 : * Returns true if kswapd is ready to sleep
7228 : */
7229 2 : static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
7230 : int highest_zoneidx)
7231 : {
7232 : /*
7233 : * The throttled processes are normally woken up in balance_pgdat() as
7234 : * soon as allow_direct_reclaim() is true. But there is a potential
7235 : * race between when kswapd checks the watermarks and a process gets
7236 : * throttled. There is also a potential race if processes get
7237 : * throttled, kswapd wakes, a large process exits thereby balancing the
7238 : * zones, which causes kswapd to exit balance_pgdat() before reaching
7239 : * the wake up checks. If kswapd is going to sleep, no process should
7240 : * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
7241 : * the wake up is premature, processes will wake kswapd and get
7242 : * throttled again. The difference from wake ups in balance_pgdat() is
7243 : * that here we are under prepare_to_wait().
7244 : */
7245 4 : if (waitqueue_active(&pgdat->pfmemalloc_wait))
7246 0 : wake_up_all(&pgdat->pfmemalloc_wait);
7247 :
7248 : /* Hopeless node, leave it to direct reclaim */
7249 2 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
7250 : return true;
7251 :
7252 2 : if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
7253 2 : clear_pgdat_congested(pgdat);
7254 2 : return true;
7255 : }
7256 :
7257 : return false;
7258 : }
7259 :
7260 : /*
7261 : * kswapd shrinks a node of pages that are at or below the highest usable
7262 : * zone that is currently unbalanced.
7263 : *
7264 : * Returns true if kswapd scanned at least the requested number of pages to
7265 : * reclaim or if the lack of progress was due to pages under writeback.
7266 : * This is used to determine if the scanning priority needs to be raised.
7267 : */
7268 0 : static bool kswapd_shrink_node(pg_data_t *pgdat,
7269 : struct scan_control *sc)
7270 : {
7271 : struct zone *zone;
7272 : int z;
7273 :
7274 : /* Reclaim a number of pages proportional to the number of zones */
7275 0 : sc->nr_to_reclaim = 0;
7276 0 : for (z = 0; z <= sc->reclaim_idx; z++) {
7277 0 : zone = pgdat->node_zones + z;
7278 0 : if (!managed_zone(zone))
7279 0 : continue;
7280 :
7281 0 : sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
7282 : }
7283 :
7284 : /*
7285 : * Historically care was taken to put equal pressure on all zones but
7286 : * now pressure is applied based on node LRU order.
7287 : */
7288 0 : shrink_node(pgdat, sc);
7289 :
7290 : /*
7291 : * Fragmentation may mean that the system cannot be rebalanced for
7292 : * high-order allocations. If twice the allocation size has been
7293 : * reclaimed then recheck watermarks only at order-0 to prevent
7294 : * excessive reclaim. Assume that a process requested a high-order
7295 : * can direct reclaim/compact.
7296 : */
7297 0 : if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
7298 0 : sc->order = 0;
7299 :
7300 0 : return sc->nr_scanned >= sc->nr_to_reclaim;
7301 : }
7302 :
7303 : /* Page allocator PCP high watermark is lowered if reclaim is active. */
7304 : static inline void
7305 : update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)
7306 : {
7307 : int i;
7308 : struct zone *zone;
7309 :
7310 0 : for (i = 0; i <= highest_zoneidx; i++) {
7311 0 : zone = pgdat->node_zones + i;
7312 :
7313 0 : if (!managed_zone(zone))
7314 0 : continue;
7315 :
7316 : if (active)
7317 0 : set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
7318 : else
7319 0 : clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
7320 : }
7321 : }
7322 :
7323 : static inline void
7324 : set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
7325 : {
7326 0 : update_reclaim_active(pgdat, highest_zoneidx, true);
7327 : }
7328 :
7329 : static inline void
7330 : clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
7331 : {
7332 0 : update_reclaim_active(pgdat, highest_zoneidx, false);
7333 : }
7334 :
7335 : /*
7336 : * For kswapd, balance_pgdat() will reclaim pages across a node from zones
7337 : * that are eligible for use by the caller until at least one zone is
7338 : * balanced.
7339 : *
7340 : * Returns the order kswapd finished reclaiming at.
7341 : *
7342 : * kswapd scans the zones in the highmem->normal->dma direction. It skips
7343 : * zones which have free_pages > high_wmark_pages(zone), but once a zone is
7344 : * found to have free_pages <= high_wmark_pages(zone), any page in that zone
7345 : * or lower is eligible for reclaim until at least one usable zone is
7346 : * balanced.
7347 : */
7348 0 : static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
7349 : {
7350 : int i;
7351 : unsigned long nr_soft_reclaimed;
7352 : unsigned long nr_soft_scanned;
7353 : unsigned long pflags;
7354 : unsigned long nr_boost_reclaim;
7355 0 : unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
7356 : bool boosted;
7357 : struct zone *zone;
7358 0 : struct scan_control sc = {
7359 : .gfp_mask = GFP_KERNEL,
7360 : .order = order,
7361 : .may_unmap = 1,
7362 : };
7363 :
7364 0 : set_task_reclaim_state(current, &sc.reclaim_state);
7365 0 : psi_memstall_enter(&pflags);
7366 0 : __fs_reclaim_acquire(_THIS_IP_);
7367 :
7368 0 : count_vm_event(PAGEOUTRUN);
7369 :
7370 : /*
7371 : * Account for the reclaim boost. Note that the zone boost is left in
7372 : * place so that parallel allocations that are near the watermark will
7373 : * stall or direct reclaim until kswapd is finished.
7374 : */
7375 0 : nr_boost_reclaim = 0;
7376 0 : for (i = 0; i <= highest_zoneidx; i++) {
7377 0 : zone = pgdat->node_zones + i;
7378 0 : if (!managed_zone(zone))
7379 0 : continue;
7380 :
7381 0 : nr_boost_reclaim += zone->watermark_boost;
7382 0 : zone_boosts[i] = zone->watermark_boost;
7383 : }
7384 : boosted = nr_boost_reclaim;
7385 :
7386 : restart:
7387 0 : set_reclaim_active(pgdat, highest_zoneidx);
7388 0 : sc.priority = DEF_PRIORITY;
7389 : do {
7390 0 : unsigned long nr_reclaimed = sc.nr_reclaimed;
7391 0 : bool raise_priority = true;
7392 : bool balanced;
7393 : bool ret;
7394 :
7395 0 : sc.reclaim_idx = highest_zoneidx;
7396 :
7397 : /*
7398 : * If the number of buffer_heads exceeds the maximum allowed
7399 : * then consider reclaiming from all zones. This has a dual
7400 : * purpose -- on 64-bit systems it is expected that
7401 : * buffer_heads are stripped during active rotation. On 32-bit
7402 : * systems, highmem pages can pin lowmem memory and shrinking
7403 : * buffers can relieve lowmem pressure. Reclaim may still not
7404 : * go ahead if all eligible zones for the original allocation
7405 : * request are balanced to avoid excessive reclaim from kswapd.
7406 : */
7407 0 : if (buffer_heads_over_limit) {
7408 0 : for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
7409 0 : zone = pgdat->node_zones + i;
7410 0 : if (!managed_zone(zone))
7411 0 : continue;
7412 :
7413 0 : sc.reclaim_idx = i;
7414 0 : break;
7415 : }
7416 : }
7417 :
7418 : /*
7419 : * If the pgdat is imbalanced then ignore boosting and preserve
7420 : * the watermarks for a later time and restart. Note that the
7421 : * zone watermarks will be still reset at the end of balancing
7422 : * on the grounds that the normal reclaim should be enough to
7423 : * re-evaluate if boosting is required when kswapd next wakes.
7424 : */
7425 0 : balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
7426 0 : if (!balanced && nr_boost_reclaim) {
7427 : nr_boost_reclaim = 0;
7428 : goto restart;
7429 : }
7430 :
7431 : /*
7432 : * If boosting is not active then only reclaim if there are no
7433 : * eligible zones. Note that sc.reclaim_idx is not used as
7434 : * buffer_heads_over_limit may have adjusted it.
7435 : */
7436 0 : if (!nr_boost_reclaim && balanced)
7437 : goto out;
7438 :
7439 : /* Limit the priority of boosting to avoid reclaim writeback */
7440 0 : if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
7441 0 : raise_priority = false;
7442 :
7443 : /*
7444 : * Do not writeback or swap pages for boosted reclaim. The
7445 : * intent is to relieve pressure not issue sub-optimal IO
7446 : * from reclaim context. If no pages are reclaimed, the
7447 : * reclaim will be aborted.
7448 : */
7449 0 : sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
7450 0 : sc.may_swap = !nr_boost_reclaim;
7451 :
7452 : /*
7453 : * Do some background aging, to give pages a chance to be
7454 : * referenced before reclaiming. All pages are rotated
7455 : * regardless of classzone as this is about consistent aging.
7456 : */
7457 0 : kswapd_age_node(pgdat, &sc);
7458 :
7459 : /*
7460 : * If we're getting trouble reclaiming, start doing writepage
7461 : * even in laptop mode.
7462 : */
7463 0 : if (sc.priority < DEF_PRIORITY - 2)
7464 0 : sc.may_writepage = 1;
7465 :
7466 : /* Call soft limit reclaim before calling shrink_node. */
7467 0 : sc.nr_scanned = 0;
7468 0 : nr_soft_scanned = 0;
7469 0 : nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
7470 : sc.gfp_mask, &nr_soft_scanned);
7471 : sc.nr_reclaimed += nr_soft_reclaimed;
7472 :
7473 : /*
7474 : * There should be no need to raise the scanning priority if
7475 : * enough pages are already being scanned that that high
7476 : * watermark would be met at 100% efficiency.
7477 : */
7478 0 : if (kswapd_shrink_node(pgdat, &sc))
7479 0 : raise_priority = false;
7480 :
7481 : /*
7482 : * If the low watermark is met there is no need for processes
7483 : * to be throttled on pfmemalloc_wait as they should not be
7484 : * able to safely make forward progress. Wake them
7485 : */
7486 0 : if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
7487 0 : allow_direct_reclaim(pgdat))
7488 0 : wake_up_all(&pgdat->pfmemalloc_wait);
7489 :
7490 : /* Check if kswapd should be suspending */
7491 0 : __fs_reclaim_release(_THIS_IP_);
7492 0 : ret = try_to_freeze();
7493 0 : __fs_reclaim_acquire(_THIS_IP_);
7494 0 : if (ret || kthread_should_stop())
7495 : break;
7496 :
7497 : /*
7498 : * Raise priority if scanning rate is too low or there was no
7499 : * progress in reclaiming pages
7500 : */
7501 0 : nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
7502 0 : nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
7503 :
7504 : /*
7505 : * If reclaim made no progress for a boost, stop reclaim as
7506 : * IO cannot be queued and it could be an infinite loop in
7507 : * extreme circumstances.
7508 : */
7509 0 : if (nr_boost_reclaim && !nr_reclaimed)
7510 : break;
7511 :
7512 0 : if (raise_priority || !nr_reclaimed)
7513 0 : sc.priority--;
7514 0 : } while (sc.priority >= 1);
7515 :
7516 0 : if (!sc.nr_reclaimed)
7517 0 : pgdat->kswapd_failures++;
7518 :
7519 : out:
7520 0 : clear_reclaim_active(pgdat, highest_zoneidx);
7521 :
7522 : /* If reclaim was boosted, account for the reclaim done in this pass */
7523 0 : if (boosted) {
7524 : unsigned long flags;
7525 :
7526 0 : for (i = 0; i <= highest_zoneidx; i++) {
7527 0 : if (!zone_boosts[i])
7528 0 : continue;
7529 :
7530 : /* Increments are under the zone lock */
7531 0 : zone = pgdat->node_zones + i;
7532 0 : spin_lock_irqsave(&zone->lock, flags);
7533 0 : zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
7534 0 : spin_unlock_irqrestore(&zone->lock, flags);
7535 : }
7536 :
7537 : /*
7538 : * As there is now likely space, wakeup kcompact to defragment
7539 : * pageblocks.
7540 : */
7541 0 : wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
7542 : }
7543 :
7544 0 : snapshot_refaults(NULL, pgdat);
7545 0 : __fs_reclaim_release(_THIS_IP_);
7546 0 : psi_memstall_leave(&pflags);
7547 0 : set_task_reclaim_state(current, NULL);
7548 :
7549 : /*
7550 : * Return the order kswapd stopped reclaiming at as
7551 : * prepare_kswapd_sleep() takes it into account. If another caller
7552 : * entered the allocator slow path while kswapd was awake, order will
7553 : * remain at the higher level.
7554 : */
7555 0 : return sc.order;
7556 : }
7557 :
7558 : /*
7559 : * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
7560 : * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
7561 : * not a valid index then either kswapd runs for first time or kswapd couldn't
7562 : * sleep after previous reclaim attempt (node is still unbalanced). In that
7563 : * case return the zone index of the previous kswapd reclaim cycle.
7564 : */
7565 : static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
7566 : enum zone_type prev_highest_zoneidx)
7567 : {
7568 1 : enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
7569 :
7570 1 : return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
7571 : }
7572 :
7573 1 : static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
7574 : unsigned int highest_zoneidx)
7575 : {
7576 1 : long remaining = 0;
7577 2 : DEFINE_WAIT(wait);
7578 :
7579 2 : if (freezing(current) || kthread_should_stop())
7580 0 : return;
7581 :
7582 1 : prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
7583 :
7584 : /*
7585 : * Try to sleep for a short interval. Note that kcompactd will only be
7586 : * woken if it is possible to sleep for a short interval. This is
7587 : * deliberate on the assumption that if reclaim cannot keep an
7588 : * eligible zone balanced that it's also unlikely that compaction will
7589 : * succeed.
7590 : */
7591 1 : if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
7592 : /*
7593 : * Compaction records what page blocks it recently failed to
7594 : * isolate pages from and skips them in the future scanning.
7595 : * When kswapd is going to sleep, it is reasonable to assume
7596 : * that pages and compaction may succeed so reset the cache.
7597 : */
7598 1 : reset_isolation_suitable(pgdat);
7599 :
7600 : /*
7601 : * We have freed the memory, now we should compact it to make
7602 : * allocation of the requested order possible.
7603 : */
7604 1 : wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
7605 :
7606 1 : remaining = schedule_timeout(HZ/10);
7607 :
7608 : /*
7609 : * If woken prematurely then reset kswapd_highest_zoneidx and
7610 : * order. The values will either be from a wakeup request or
7611 : * the previous request that slept prematurely.
7612 : */
7613 1 : if (remaining) {
7614 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
7615 : kswapd_highest_zoneidx(pgdat,
7616 : highest_zoneidx));
7617 :
7618 0 : if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
7619 0 : WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
7620 : }
7621 :
7622 1 : finish_wait(&pgdat->kswapd_wait, &wait);
7623 1 : prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
7624 : }
7625 :
7626 : /*
7627 : * After a short sleep, check if it was a premature sleep. If not, then
7628 : * go fully to sleep until explicitly woken up.
7629 : */
7630 2 : if (!remaining &&
7631 1 : prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
7632 1 : trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
7633 :
7634 : /*
7635 : * vmstat counters are not perfectly accurate and the estimated
7636 : * value for counters such as NR_FREE_PAGES can deviate from the
7637 : * true value by nr_online_cpus * threshold. To avoid the zone
7638 : * watermarks being breached while under pressure, we reduce the
7639 : * per-cpu vmstat threshold while kswapd is awake and restore
7640 : * them before going back to sleep.
7641 : */
7642 : set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
7643 :
7644 1 : if (!kthread_should_stop())
7645 1 : schedule();
7646 :
7647 : set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
7648 : } else {
7649 0 : if (remaining)
7650 0 : count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
7651 : else
7652 0 : count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
7653 : }
7654 0 : finish_wait(&pgdat->kswapd_wait, &wait);
7655 : }
7656 :
7657 : /*
7658 : * The background pageout daemon, started as a kernel thread
7659 : * from the init process.
7660 : *
7661 : * This basically trickles out pages so that we have _some_
7662 : * free memory available even if there is no other activity
7663 : * that frees anything up. This is needed for things like routing
7664 : * etc, where we otherwise might have all activity going on in
7665 : * asynchronous contexts that cannot page things out.
7666 : *
7667 : * If there are applications that are active memory-allocators
7668 : * (most normal use), this basically shouldn't matter.
7669 : */
7670 1 : static int kswapd(void *p)
7671 : {
7672 : unsigned int alloc_order, reclaim_order;
7673 1 : unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
7674 1 : pg_data_t *pgdat = (pg_data_t *)p;
7675 1 : struct task_struct *tsk = current;
7676 1 : const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
7677 :
7678 1 : if (!cpumask_empty(cpumask))
7679 1 : set_cpus_allowed_ptr(tsk, cpumask);
7680 :
7681 : /*
7682 : * Tell the memory management that we're a "memory allocator",
7683 : * and that if we need more memory we should get access to it
7684 : * regardless (see "__alloc_pages()"). "kswapd" should
7685 : * never get caught in the normal page freeing logic.
7686 : *
7687 : * (Kswapd normally doesn't need memory anyway, but sometimes
7688 : * you need a small amount of memory in order to be able to
7689 : * page out something else, and this flag essentially protects
7690 : * us from recursively trying to free more memory as we're
7691 : * trying to free the first piece of memory in the first place).
7692 : */
7693 1 : tsk->flags |= PF_MEMALLOC | PF_KSWAPD;
7694 1 : set_freezable();
7695 :
7696 1 : WRITE_ONCE(pgdat->kswapd_order, 0);
7697 1 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
7698 1 : atomic_set(&pgdat->nr_writeback_throttled, 0);
7699 : for ( ; ; ) {
7700 : bool ret;
7701 :
7702 1 : alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
7703 : highest_zoneidx = kswapd_highest_zoneidx(pgdat,
7704 : highest_zoneidx);
7705 :
7706 : kswapd_try_sleep:
7707 1 : kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
7708 : highest_zoneidx);
7709 :
7710 : /* Read the new order and highest_zoneidx */
7711 0 : alloc_order = READ_ONCE(pgdat->kswapd_order);
7712 0 : highest_zoneidx = kswapd_highest_zoneidx(pgdat,
7713 : highest_zoneidx);
7714 0 : WRITE_ONCE(pgdat->kswapd_order, 0);
7715 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
7716 :
7717 0 : ret = try_to_freeze();
7718 0 : if (kthread_should_stop())
7719 : break;
7720 :
7721 : /*
7722 : * We can speed up thawing tasks if we don't call balance_pgdat
7723 : * after returning from the refrigerator
7724 : */
7725 0 : if (ret)
7726 0 : continue;
7727 :
7728 : /*
7729 : * Reclaim begins at the requested order but if a high-order
7730 : * reclaim fails then kswapd falls back to reclaiming for
7731 : * order-0. If that happens, kswapd will consider sleeping
7732 : * for the order it finished reclaiming at (reclaim_order)
7733 : * but kcompactd is woken to compact for the original
7734 : * request (alloc_order).
7735 : */
7736 0 : trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
7737 : alloc_order);
7738 0 : reclaim_order = balance_pgdat(pgdat, alloc_order,
7739 : highest_zoneidx);
7740 0 : if (reclaim_order < alloc_order)
7741 : goto kswapd_try_sleep;
7742 : }
7743 :
7744 0 : tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD);
7745 :
7746 0 : return 0;
7747 : }
7748 :
7749 : /*
7750 : * A zone is low on free memory or too fragmented for high-order memory. If
7751 : * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
7752 : * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim
7753 : * has failed or is not needed, still wake up kcompactd if only compaction is
7754 : * needed.
7755 : */
7756 0 : void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
7757 : enum zone_type highest_zoneidx)
7758 : {
7759 : pg_data_t *pgdat;
7760 : enum zone_type curr_idx;
7761 :
7762 0 : if (!managed_zone(zone))
7763 : return;
7764 :
7765 0 : if (!cpuset_zone_allowed(zone, gfp_flags))
7766 : return;
7767 :
7768 0 : pgdat = zone->zone_pgdat;
7769 0 : curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
7770 :
7771 0 : if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx)
7772 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
7773 :
7774 0 : if (READ_ONCE(pgdat->kswapd_order) < order)
7775 0 : WRITE_ONCE(pgdat->kswapd_order, order);
7776 :
7777 0 : if (!waitqueue_active(&pgdat->kswapd_wait))
7778 : return;
7779 :
7780 : /* Hopeless node, leave it to direct reclaim if possible */
7781 0 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
7782 0 : (pgdat_balanced(pgdat, order, highest_zoneidx) &&
7783 0 : !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
7784 : /*
7785 : * There may be plenty of free memory available, but it's too
7786 : * fragmented for high-order allocations. Wake up kcompactd
7787 : * and rely on compaction_suitable() to determine if it's
7788 : * needed. If it fails, it will defer subsequent attempts to
7789 : * ratelimit its work.
7790 : */
7791 0 : if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
7792 0 : wakeup_kcompactd(pgdat, order, highest_zoneidx);
7793 : return;
7794 : }
7795 :
7796 0 : trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
7797 : gfp_flags);
7798 0 : wake_up_interruptible(&pgdat->kswapd_wait);
7799 : }
7800 :
7801 : #ifdef CONFIG_HIBERNATION
7802 : /*
7803 : * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
7804 : * freed pages.
7805 : *
7806 : * Rather than trying to age LRUs the aim is to preserve the overall
7807 : * LRU order by reclaiming preferentially
7808 : * inactive > active > active referenced > active mapped
7809 : */
7810 : unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
7811 : {
7812 : struct scan_control sc = {
7813 : .nr_to_reclaim = nr_to_reclaim,
7814 : .gfp_mask = GFP_HIGHUSER_MOVABLE,
7815 : .reclaim_idx = MAX_NR_ZONES - 1,
7816 : .priority = DEF_PRIORITY,
7817 : .may_writepage = 1,
7818 : .may_unmap = 1,
7819 : .may_swap = 1,
7820 : .hibernation_mode = 1,
7821 : };
7822 : struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
7823 : unsigned long nr_reclaimed;
7824 : unsigned int noreclaim_flag;
7825 :
7826 : fs_reclaim_acquire(sc.gfp_mask);
7827 : noreclaim_flag = memalloc_noreclaim_save();
7828 : set_task_reclaim_state(current, &sc.reclaim_state);
7829 :
7830 : nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
7831 :
7832 : set_task_reclaim_state(current, NULL);
7833 : memalloc_noreclaim_restore(noreclaim_flag);
7834 : fs_reclaim_release(sc.gfp_mask);
7835 :
7836 : return nr_reclaimed;
7837 : }
7838 : #endif /* CONFIG_HIBERNATION */
7839 :
7840 : /*
7841 : * This kswapd start function will be called by init and node-hot-add.
7842 : */
7843 1 : void kswapd_run(int nid)
7844 : {
7845 1 : pg_data_t *pgdat = NODE_DATA(nid);
7846 :
7847 : pgdat_kswapd_lock(pgdat);
7848 1 : if (!pgdat->kswapd) {
7849 2 : pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
7850 1 : if (IS_ERR(pgdat->kswapd)) {
7851 : /* failure at boot is fatal */
7852 0 : BUG_ON(system_state < SYSTEM_RUNNING);
7853 0 : pr_err("Failed to start kswapd on node %d\n", nid);
7854 0 : pgdat->kswapd = NULL;
7855 : }
7856 : }
7857 : pgdat_kswapd_unlock(pgdat);
7858 1 : }
7859 :
7860 : /*
7861 : * Called by memory hotplug when all memory in a node is offlined. Caller must
7862 : * be holding mem_hotplug_begin/done().
7863 : */
7864 0 : void kswapd_stop(int nid)
7865 : {
7866 0 : pg_data_t *pgdat = NODE_DATA(nid);
7867 : struct task_struct *kswapd;
7868 :
7869 : pgdat_kswapd_lock(pgdat);
7870 0 : kswapd = pgdat->kswapd;
7871 0 : if (kswapd) {
7872 0 : kthread_stop(kswapd);
7873 0 : pgdat->kswapd = NULL;
7874 : }
7875 : pgdat_kswapd_unlock(pgdat);
7876 0 : }
7877 :
7878 1 : static int __init kswapd_init(void)
7879 : {
7880 : int nid;
7881 :
7882 1 : swap_setup();
7883 2 : for_each_node_state(nid, N_MEMORY)
7884 1 : kswapd_run(nid);
7885 1 : return 0;
7886 : }
7887 :
7888 : module_init(kswapd_init)
7889 :
7890 : #ifdef CONFIG_NUMA
7891 : /*
7892 : * Node reclaim mode
7893 : *
7894 : * If non-zero call node_reclaim when the number of free pages falls below
7895 : * the watermarks.
7896 : */
7897 : int node_reclaim_mode __read_mostly;
7898 :
7899 : /*
7900 : * Priority for NODE_RECLAIM. This determines the fraction of pages
7901 : * of a node considered for each zone_reclaim. 4 scans 1/16th of
7902 : * a zone.
7903 : */
7904 : #define NODE_RECLAIM_PRIORITY 4
7905 :
7906 : /*
7907 : * Percentage of pages in a zone that must be unmapped for node_reclaim to
7908 : * occur.
7909 : */
7910 : int sysctl_min_unmapped_ratio = 1;
7911 :
7912 : /*
7913 : * If the number of slab pages in a zone grows beyond this percentage then
7914 : * slab reclaim needs to occur.
7915 : */
7916 : int sysctl_min_slab_ratio = 5;
7917 :
7918 : static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
7919 : {
7920 : unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
7921 : unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
7922 : node_page_state(pgdat, NR_ACTIVE_FILE);
7923 :
7924 : /*
7925 : * It's possible for there to be more file mapped pages than
7926 : * accounted for by the pages on the file LRU lists because
7927 : * tmpfs pages accounted for as ANON can also be FILE_MAPPED
7928 : */
7929 : return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
7930 : }
7931 :
7932 : /* Work out how many page cache pages we can reclaim in this reclaim_mode */
7933 : static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
7934 : {
7935 : unsigned long nr_pagecache_reclaimable;
7936 : unsigned long delta = 0;
7937 :
7938 : /*
7939 : * If RECLAIM_UNMAP is set, then all file pages are considered
7940 : * potentially reclaimable. Otherwise, we have to worry about
7941 : * pages like swapcache and node_unmapped_file_pages() provides
7942 : * a better estimate
7943 : */
7944 : if (node_reclaim_mode & RECLAIM_UNMAP)
7945 : nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
7946 : else
7947 : nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
7948 :
7949 : /* If we can't clean pages, remove dirty pages from consideration */
7950 : if (!(node_reclaim_mode & RECLAIM_WRITE))
7951 : delta += node_page_state(pgdat, NR_FILE_DIRTY);
7952 :
7953 : /* Watch for any possible underflows due to delta */
7954 : if (unlikely(delta > nr_pagecache_reclaimable))
7955 : delta = nr_pagecache_reclaimable;
7956 :
7957 : return nr_pagecache_reclaimable - delta;
7958 : }
7959 :
7960 : /*
7961 : * Try to free up some pages from this node through reclaim.
7962 : */
7963 : static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
7964 : {
7965 : /* Minimum pages needed in order to stay on node */
7966 : const unsigned long nr_pages = 1 << order;
7967 : struct task_struct *p = current;
7968 : unsigned int noreclaim_flag;
7969 : struct scan_control sc = {
7970 : .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
7971 : .gfp_mask = current_gfp_context(gfp_mask),
7972 : .order = order,
7973 : .priority = NODE_RECLAIM_PRIORITY,
7974 : .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
7975 : .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
7976 : .may_swap = 1,
7977 : .reclaim_idx = gfp_zone(gfp_mask),
7978 : };
7979 : unsigned long pflags;
7980 :
7981 : trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
7982 : sc.gfp_mask);
7983 :
7984 : cond_resched();
7985 : psi_memstall_enter(&pflags);
7986 : fs_reclaim_acquire(sc.gfp_mask);
7987 : /*
7988 : * We need to be able to allocate from the reserves for RECLAIM_UNMAP
7989 : */
7990 : noreclaim_flag = memalloc_noreclaim_save();
7991 : set_task_reclaim_state(p, &sc.reclaim_state);
7992 :
7993 : if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages ||
7994 : node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) {
7995 : /*
7996 : * Free memory by calling shrink node with increasing
7997 : * priorities until we have enough memory freed.
7998 : */
7999 : do {
8000 : shrink_node(pgdat, &sc);
8001 : } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
8002 : }
8003 :
8004 : set_task_reclaim_state(p, NULL);
8005 : memalloc_noreclaim_restore(noreclaim_flag);
8006 : fs_reclaim_release(sc.gfp_mask);
8007 : psi_memstall_leave(&pflags);
8008 :
8009 : trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);
8010 :
8011 : return sc.nr_reclaimed >= nr_pages;
8012 : }
8013 :
8014 : int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
8015 : {
8016 : int ret;
8017 :
8018 : /*
8019 : * Node reclaim reclaims unmapped file backed pages and
8020 : * slab pages if we are over the defined limits.
8021 : *
8022 : * A small portion of unmapped file backed pages is needed for
8023 : * file I/O otherwise pages read by file I/O will be immediately
8024 : * thrown out if the node is overallocated. So we do not reclaim
8025 : * if less than a specified percentage of the node is used by
8026 : * unmapped file backed pages.
8027 : */
8028 : if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
8029 : node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
8030 : pgdat->min_slab_pages)
8031 : return NODE_RECLAIM_FULL;
8032 :
8033 : /*
8034 : * Do not scan if the allocation should not be delayed.
8035 : */
8036 : if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
8037 : return NODE_RECLAIM_NOSCAN;
8038 :
8039 : /*
8040 : * Only run node reclaim on the local node or on nodes that do not
8041 : * have associated processors. This will favor the local processor
8042 : * over remote processors and spread off node memory allocations
8043 : * as wide as possible.
8044 : */
8045 : if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
8046 : return NODE_RECLAIM_NOSCAN;
8047 :
8048 : if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
8049 : return NODE_RECLAIM_NOSCAN;
8050 :
8051 : ret = __node_reclaim(pgdat, gfp_mask, order);
8052 : clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
8053 :
8054 : if (!ret)
8055 : count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
8056 :
8057 : return ret;
8058 : }
8059 : #endif
8060 :
8061 0 : void check_move_unevictable_pages(struct pagevec *pvec)
8062 : {
8063 : struct folio_batch fbatch;
8064 : unsigned i;
8065 :
8066 0 : folio_batch_init(&fbatch);
8067 0 : for (i = 0; i < pvec->nr; i++) {
8068 0 : struct page *page = pvec->pages[i];
8069 :
8070 0 : if (PageTransTail(page))
8071 : continue;
8072 0 : folio_batch_add(&fbatch, page_folio(page));
8073 : }
8074 0 : check_move_unevictable_folios(&fbatch);
8075 0 : }
8076 : EXPORT_SYMBOL_GPL(check_move_unevictable_pages);
8077 :
8078 : /**
8079 : * check_move_unevictable_folios - Move evictable folios to appropriate zone
8080 : * lru list
8081 : * @fbatch: Batch of lru folios to check.
8082 : *
8083 : * Checks folios for evictability, if an evictable folio is in the unevictable
8084 : * lru list, moves it to the appropriate evictable lru list. This function
8085 : * should be only used for lru folios.
8086 : */
8087 0 : void check_move_unevictable_folios(struct folio_batch *fbatch)
8088 : {
8089 0 : struct lruvec *lruvec = NULL;
8090 0 : int pgscanned = 0;
8091 0 : int pgrescued = 0;
8092 : int i;
8093 :
8094 0 : for (i = 0; i < fbatch->nr; i++) {
8095 0 : struct folio *folio = fbatch->folios[i];
8096 0 : int nr_pages = folio_nr_pages(folio);
8097 :
8098 0 : pgscanned += nr_pages;
8099 :
8100 : /* block memcg migration while the folio moves between lrus */
8101 0 : if (!folio_test_clear_lru(folio))
8102 0 : continue;
8103 :
8104 0 : lruvec = folio_lruvec_relock_irq(folio, lruvec);
8105 0 : if (folio_evictable(folio) && folio_test_unevictable(folio)) {
8106 0 : lruvec_del_folio(lruvec, folio);
8107 0 : folio_clear_unevictable(folio);
8108 0 : lruvec_add_folio(lruvec, folio);
8109 0 : pgrescued += nr_pages;
8110 : }
8111 : folio_set_lru(folio);
8112 : }
8113 :
8114 0 : if (lruvec) {
8115 0 : __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
8116 0 : __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
8117 0 : unlock_page_lruvec_irq(lruvec);
8118 0 : } else if (pgscanned) {
8119 0 : count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
8120 : }
8121 0 : }
8122 : EXPORT_SYMBOL_GPL(check_move_unevictable_folios);
|
