Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
4 : *
5 : * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
6 : *
7 : * Interactivity improvements by Mike Galbraith
8 : * (C) 2007 Mike Galbraith <efault@gmx.de>
9 : *
10 : * Various enhancements by Dmitry Adamushko.
11 : * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
12 : *
13 : * Group scheduling enhancements by Srivatsa Vaddagiri
14 : * Copyright IBM Corporation, 2007
15 : * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
16 : *
17 : * Scaled math optimizations by Thomas Gleixner
18 : * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
19 : *
20 : * Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21 : * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
22 : */
23 : #include <linux/energy_model.h>
24 : #include <linux/mmap_lock.h>
25 : #include <linux/hugetlb_inline.h>
26 : #include <linux/jiffies.h>
27 : #include <linux/mm_api.h>
28 : #include <linux/highmem.h>
29 : #include <linux/spinlock_api.h>
30 : #include <linux/cpumask_api.h>
31 : #include <linux/lockdep_api.h>
32 : #include <linux/softirq.h>
33 : #include <linux/refcount_api.h>
34 : #include <linux/topology.h>
35 : #include <linux/sched/clock.h>
36 : #include <linux/sched/cond_resched.h>
37 : #include <linux/sched/cputime.h>
38 : #include <linux/sched/isolation.h>
39 : #include <linux/sched/nohz.h>
40 :
41 : #include <linux/cpuidle.h>
42 : #include <linux/interrupt.h>
43 : #include <linux/memory-tiers.h>
44 : #include <linux/mempolicy.h>
45 : #include <linux/mutex_api.h>
46 : #include <linux/profile.h>
47 : #include <linux/psi.h>
48 : #include <linux/ratelimit.h>
49 : #include <linux/task_work.h>
50 :
51 : #include <asm/switch_to.h>
52 :
53 : #include <linux/sched/cond_resched.h>
54 :
55 : #include "sched.h"
56 : #include "stats.h"
57 : #include "autogroup.h"
58 :
59 : /*
60 : * Targeted preemption latency for CPU-bound tasks:
61 : *
62 : * NOTE: this latency value is not the same as the concept of
63 : * 'timeslice length' - timeslices in CFS are of variable length
64 : * and have no persistent notion like in traditional, time-slice
65 : * based scheduling concepts.
66 : *
67 : * (to see the precise effective timeslice length of your workload,
68 : * run vmstat and monitor the context-switches (cs) field)
69 : *
70 : * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
71 : */
72 : unsigned int sysctl_sched_latency = 6000000ULL;
73 : static unsigned int normalized_sysctl_sched_latency = 6000000ULL;
74 :
75 : /*
76 : * The initial- and re-scaling of tunables is configurable
77 : *
78 : * Options are:
79 : *
80 : * SCHED_TUNABLESCALING_NONE - unscaled, always *1
81 : * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
82 : * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
83 : *
84 : * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
85 : */
86 : unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
87 :
88 : /*
89 : * Minimal preemption granularity for CPU-bound tasks:
90 : *
91 : * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
92 : */
93 : unsigned int sysctl_sched_min_granularity = 750000ULL;
94 : static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;
95 :
96 : /*
97 : * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks.
98 : * Applies only when SCHED_IDLE tasks compete with normal tasks.
99 : *
100 : * (default: 0.75 msec)
101 : */
102 : unsigned int sysctl_sched_idle_min_granularity = 750000ULL;
103 :
104 : /*
105 : * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity
106 : */
107 : static unsigned int sched_nr_latency = 8;
108 :
109 : /*
110 : * After fork, child runs first. If set to 0 (default) then
111 : * parent will (try to) run first.
112 : */
113 : unsigned int sysctl_sched_child_runs_first __read_mostly;
114 :
115 : /*
116 : * SCHED_OTHER wake-up granularity.
117 : *
118 : * This option delays the preemption effects of decoupled workloads
119 : * and reduces their over-scheduling. Synchronous workloads will still
120 : * have immediate wakeup/sleep latencies.
121 : *
122 : * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
123 : */
124 : unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
125 : static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
126 :
127 : const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
128 :
129 : int sched_thermal_decay_shift;
130 0 : static int __init setup_sched_thermal_decay_shift(char *str)
131 : {
132 0 : int _shift = 0;
133 :
134 0 : if (kstrtoint(str, 0, &_shift))
135 0 : pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n");
136 :
137 0 : sched_thermal_decay_shift = clamp(_shift, 0, 10);
138 0 : return 1;
139 : }
140 : __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift);
141 :
142 : #ifdef CONFIG_SMP
143 : /*
144 : * For asym packing, by default the lower numbered CPU has higher priority.
145 : */
146 : int __weak arch_asym_cpu_priority(int cpu)
147 : {
148 : return -cpu;
149 : }
150 :
151 : /*
152 : * The margin used when comparing utilization with CPU capacity.
153 : *
154 : * (default: ~20%)
155 : */
156 : #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024)
157 :
158 : /*
159 : * The margin used when comparing CPU capacities.
160 : * is 'cap1' noticeably greater than 'cap2'
161 : *
162 : * (default: ~5%)
163 : */
164 : #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078)
165 : #endif
166 :
167 : #ifdef CONFIG_CFS_BANDWIDTH
168 : /*
169 : * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
170 : * each time a cfs_rq requests quota.
171 : *
172 : * Note: in the case that the slice exceeds the runtime remaining (either due
173 : * to consumption or the quota being specified to be smaller than the slice)
174 : * we will always only issue the remaining available time.
175 : *
176 : * (default: 5 msec, units: microseconds)
177 : */
178 : static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
179 : #endif
180 :
181 : #ifdef CONFIG_NUMA_BALANCING
182 : /* Restrict the NUMA promotion throughput (MB/s) for each target node. */
183 : static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
184 : #endif
185 :
186 : #ifdef CONFIG_SYSCTL
187 : static struct ctl_table sched_fair_sysctls[] = {
188 : {
189 : .procname = "sched_child_runs_first",
190 : .data = &sysctl_sched_child_runs_first,
191 : .maxlen = sizeof(unsigned int),
192 : .mode = 0644,
193 : .proc_handler = proc_dointvec,
194 : },
195 : #ifdef CONFIG_CFS_BANDWIDTH
196 : {
197 : .procname = "sched_cfs_bandwidth_slice_us",
198 : .data = &sysctl_sched_cfs_bandwidth_slice,
199 : .maxlen = sizeof(unsigned int),
200 : .mode = 0644,
201 : .proc_handler = proc_dointvec_minmax,
202 : .extra1 = SYSCTL_ONE,
203 : },
204 : #endif
205 : #ifdef CONFIG_NUMA_BALANCING
206 : {
207 : .procname = "numa_balancing_promote_rate_limit_MBps",
208 : .data = &sysctl_numa_balancing_promote_rate_limit,
209 : .maxlen = sizeof(unsigned int),
210 : .mode = 0644,
211 : .proc_handler = proc_dointvec_minmax,
212 : .extra1 = SYSCTL_ZERO,
213 : },
214 : #endif /* CONFIG_NUMA_BALANCING */
215 : {}
216 : };
217 :
218 1 : static int __init sched_fair_sysctl_init(void)
219 : {
220 1 : register_sysctl_init("kernel", sched_fair_sysctls);
221 1 : return 0;
222 : }
223 : late_initcall(sched_fair_sysctl_init);
224 : #endif
225 :
226 : static inline void update_load_add(struct load_weight *lw, unsigned long inc)
227 : {
228 2472 : lw->weight += inc;
229 2472 : lw->inv_weight = 0;
230 : }
231 :
232 : static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
233 : {
234 2130 : lw->weight -= dec;
235 2130 : lw->inv_weight = 0;
236 : }
237 :
238 : static inline void update_load_set(struct load_weight *lw, unsigned long w)
239 : {
240 5 : lw->weight = w;
241 5 : lw->inv_weight = 0;
242 : }
243 :
244 : /*
245 : * Increase the granularity value when there are more CPUs,
246 : * because with more CPUs the 'effective latency' as visible
247 : * to users decreases. But the relationship is not linear,
248 : * so pick a second-best guess by going with the log2 of the
249 : * number of CPUs.
250 : *
251 : * This idea comes from the SD scheduler of Con Kolivas:
252 : */
253 : static unsigned int get_update_sysctl_factor(void)
254 : {
255 1 : unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
256 : unsigned int factor;
257 :
258 : switch (sysctl_sched_tunable_scaling) {
259 : case SCHED_TUNABLESCALING_NONE:
260 : factor = 1;
261 : break;
262 : case SCHED_TUNABLESCALING_LINEAR:
263 : factor = cpus;
264 : break;
265 : case SCHED_TUNABLESCALING_LOG:
266 : default:
267 : factor = 1 + ilog2(cpus);
268 : break;
269 : }
270 :
271 : return factor;
272 : }
273 :
274 : static void update_sysctl(void)
275 : {
276 1 : unsigned int factor = get_update_sysctl_factor();
277 :
278 : #define SET_SYSCTL(name) \
279 : (sysctl_##name = (factor) * normalized_sysctl_##name)
280 1 : SET_SYSCTL(sched_min_granularity);
281 1 : SET_SYSCTL(sched_latency);
282 1 : SET_SYSCTL(sched_wakeup_granularity);
283 : #undef SET_SYSCTL
284 : }
285 :
286 1 : void __init sched_init_granularity(void)
287 : {
288 : update_sysctl();
289 1 : }
290 :
291 : #define WMULT_CONST (~0U)
292 : #define WMULT_SHIFT 32
293 :
294 : static void __update_inv_weight(struct load_weight *lw)
295 : {
296 : unsigned long w;
297 :
298 465 : if (likely(lw->inv_weight))
299 : return;
300 :
301 390 : w = scale_load_down(lw->weight);
302 :
303 390 : if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
304 0 : lw->inv_weight = 1;
305 390 : else if (unlikely(!w))
306 0 : lw->inv_weight = WMULT_CONST;
307 : else
308 390 : lw->inv_weight = WMULT_CONST / w;
309 : }
310 :
311 : /*
312 : * delta_exec * weight / lw.weight
313 : * OR
314 : * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
315 : *
316 : * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case
317 : * we're guaranteed shift stays positive because inv_weight is guaranteed to
318 : * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
319 : *
320 : * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
321 : * weight/lw.weight <= 1, and therefore our shift will also be positive.
322 : */
323 465 : static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
324 : {
325 465 : u64 fact = scale_load_down(weight);
326 465 : u32 fact_hi = (u32)(fact >> 32);
327 465 : int shift = WMULT_SHIFT;
328 : int fs;
329 :
330 465 : __update_inv_weight(lw);
331 :
332 465 : if (unlikely(fact_hi)) {
333 0 : fs = fls(fact_hi);
334 0 : shift -= fs;
335 0 : fact >>= fs;
336 : }
337 :
338 930 : fact = mul_u32_u32(fact, lw->inv_weight);
339 :
340 465 : fact_hi = (u32)(fact >> 32);
341 465 : if (fact_hi) {
342 0 : fs = fls(fact_hi);
343 0 : shift -= fs;
344 0 : fact >>= fs;
345 : }
346 :
347 930 : return mul_u64_u32_shr(delta_exec, fact, shift);
348 : }
349 :
350 :
351 : const struct sched_class fair_sched_class;
352 :
353 : /**************************************************************
354 : * CFS operations on generic schedulable entities:
355 : */
356 :
357 : #ifdef CONFIG_FAIR_GROUP_SCHED
358 :
359 : /* Walk up scheduling entities hierarchy */
360 : #define for_each_sched_entity(se) \
361 : for (; se; se = se->parent)
362 :
363 : static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
364 : {
365 : struct rq *rq = rq_of(cfs_rq);
366 : int cpu = cpu_of(rq);
367 :
368 : if (cfs_rq->on_list)
369 : return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list;
370 :
371 : cfs_rq->on_list = 1;
372 :
373 : /*
374 : * Ensure we either appear before our parent (if already
375 : * enqueued) or force our parent to appear after us when it is
376 : * enqueued. The fact that we always enqueue bottom-up
377 : * reduces this to two cases and a special case for the root
378 : * cfs_rq. Furthermore, it also means that we will always reset
379 : * tmp_alone_branch either when the branch is connected
380 : * to a tree or when we reach the top of the tree
381 : */
382 : if (cfs_rq->tg->parent &&
383 : cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
384 : /*
385 : * If parent is already on the list, we add the child
386 : * just before. Thanks to circular linked property of
387 : * the list, this means to put the child at the tail
388 : * of the list that starts by parent.
389 : */
390 : list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
391 : &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
392 : /*
393 : * The branch is now connected to its tree so we can
394 : * reset tmp_alone_branch to the beginning of the
395 : * list.
396 : */
397 : rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
398 : return true;
399 : }
400 :
401 : if (!cfs_rq->tg->parent) {
402 : /*
403 : * cfs rq without parent should be put
404 : * at the tail of the list.
405 : */
406 : list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
407 : &rq->leaf_cfs_rq_list);
408 : /*
409 : * We have reach the top of a tree so we can reset
410 : * tmp_alone_branch to the beginning of the list.
411 : */
412 : rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
413 : return true;
414 : }
415 :
416 : /*
417 : * The parent has not already been added so we want to
418 : * make sure that it will be put after us.
419 : * tmp_alone_branch points to the begin of the branch
420 : * where we will add parent.
421 : */
422 : list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch);
423 : /*
424 : * update tmp_alone_branch to points to the new begin
425 : * of the branch
426 : */
427 : rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list;
428 : return false;
429 : }
430 :
431 : static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
432 : {
433 : if (cfs_rq->on_list) {
434 : struct rq *rq = rq_of(cfs_rq);
435 :
436 : /*
437 : * With cfs_rq being unthrottled/throttled during an enqueue,
438 : * it can happen the tmp_alone_branch points the a leaf that
439 : * we finally want to del. In this case, tmp_alone_branch moves
440 : * to the prev element but it will point to rq->leaf_cfs_rq_list
441 : * at the end of the enqueue.
442 : */
443 : if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list)
444 : rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev;
445 :
446 : list_del_rcu(&cfs_rq->leaf_cfs_rq_list);
447 : cfs_rq->on_list = 0;
448 : }
449 : }
450 :
451 : static inline void assert_list_leaf_cfs_rq(struct rq *rq)
452 : {
453 : SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list);
454 : }
455 :
456 : /* Iterate thr' all leaf cfs_rq's on a runqueue */
457 : #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
458 : list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \
459 : leaf_cfs_rq_list)
460 :
461 : /* Do the two (enqueued) entities belong to the same group ? */
462 : static inline struct cfs_rq *
463 : is_same_group(struct sched_entity *se, struct sched_entity *pse)
464 : {
465 : if (se->cfs_rq == pse->cfs_rq)
466 : return se->cfs_rq;
467 :
468 : return NULL;
469 : }
470 :
471 : static inline struct sched_entity *parent_entity(const struct sched_entity *se)
472 : {
473 : return se->parent;
474 : }
475 :
476 : static void
477 : find_matching_se(struct sched_entity **se, struct sched_entity **pse)
478 : {
479 : int se_depth, pse_depth;
480 :
481 : /*
482 : * preemption test can be made between sibling entities who are in the
483 : * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
484 : * both tasks until we find their ancestors who are siblings of common
485 : * parent.
486 : */
487 :
488 : /* First walk up until both entities are at same depth */
489 : se_depth = (*se)->depth;
490 : pse_depth = (*pse)->depth;
491 :
492 : while (se_depth > pse_depth) {
493 : se_depth--;
494 : *se = parent_entity(*se);
495 : }
496 :
497 : while (pse_depth > se_depth) {
498 : pse_depth--;
499 : *pse = parent_entity(*pse);
500 : }
501 :
502 : while (!is_same_group(*se, *pse)) {
503 : *se = parent_entity(*se);
504 : *pse = parent_entity(*pse);
505 : }
506 : }
507 :
508 : static int tg_is_idle(struct task_group *tg)
509 : {
510 : return tg->idle > 0;
511 : }
512 :
513 : static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
514 : {
515 : return cfs_rq->idle > 0;
516 : }
517 :
518 : static int se_is_idle(struct sched_entity *se)
519 : {
520 : if (entity_is_task(se))
521 : return task_has_idle_policy(task_of(se));
522 : return cfs_rq_is_idle(group_cfs_rq(se));
523 : }
524 :
525 : #else /* !CONFIG_FAIR_GROUP_SCHED */
526 :
527 : #define for_each_sched_entity(se) \
528 : for (; se; se = NULL)
529 :
530 : static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
531 : {
532 : return true;
533 : }
534 :
535 : static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
536 : {
537 : }
538 :
539 : static inline void assert_list_leaf_cfs_rq(struct rq *rq)
540 : {
541 : }
542 :
543 : #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
544 : for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos)
545 :
546 : static inline struct sched_entity *parent_entity(struct sched_entity *se)
547 : {
548 : return NULL;
549 : }
550 :
551 : static inline void
552 : find_matching_se(struct sched_entity **se, struct sched_entity **pse)
553 : {
554 : }
555 :
556 : static inline int tg_is_idle(struct task_group *tg)
557 : {
558 : return 0;
559 : }
560 :
561 : static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
562 : {
563 : return 0;
564 : }
565 :
566 : static int se_is_idle(struct sched_entity *se)
567 : {
568 : return 0;
569 : }
570 :
571 : #endif /* CONFIG_FAIR_GROUP_SCHED */
572 :
573 : static __always_inline
574 : void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
575 :
576 : /**************************************************************
577 : * Scheduling class tree data structure manipulation methods:
578 : */
579 :
580 : static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
581 : {
582 6973 : s64 delta = (s64)(vruntime - max_vruntime);
583 6973 : if (delta > 0)
584 4434 : max_vruntime = vruntime;
585 :
586 : return max_vruntime;
587 : }
588 :
589 : static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
590 : {
591 125 : s64 delta = (s64)(vruntime - min_vruntime);
592 125 : if (delta < 0)
593 125 : min_vruntime = vruntime;
594 :
595 : return min_vruntime;
596 : }
597 :
598 : static inline bool entity_before(const struct sched_entity *a,
599 : const struct sched_entity *b)
600 : {
601 842 : return (s64)(a->vruntime - b->vruntime) < 0;
602 : }
603 :
604 : #define __node_2_se(node) \
605 : rb_entry((node), struct sched_entity, run_node)
606 :
607 4845 : static void update_min_vruntime(struct cfs_rq *cfs_rq)
608 : {
609 4845 : struct sched_entity *curr = cfs_rq->curr;
610 4845 : struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline);
611 :
612 4845 : u64 vruntime = cfs_rq->min_vruntime;
613 :
614 4845 : if (curr) {
615 4845 : if (curr->on_rq)
616 2719 : vruntime = curr->vruntime;
617 : else
618 : curr = NULL;
619 : }
620 :
621 4845 : if (leftmost) { /* non-empty tree */
622 2250 : struct sched_entity *se = __node_2_se(leftmost);
623 :
624 2250 : if (!curr)
625 2125 : vruntime = se->vruntime;
626 : else
627 125 : vruntime = min_vruntime(vruntime, se->vruntime);
628 : }
629 :
630 : /* ensure we never gain time by being placed backwards. */
631 9690 : u64_u32_store(cfs_rq->min_vruntime,
632 : max_vruntime(cfs_rq->min_vruntime, vruntime));
633 4845 : }
634 :
635 : static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
636 : {
637 842 : return entity_before(__node_2_se(a), __node_2_se(b));
638 : }
639 :
640 : /*
641 : * Enqueue an entity into the rb-tree:
642 : */
643 2214 : static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
644 : {
645 4428 : rb_add_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less);
646 2214 : }
647 :
648 : static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
649 : {
650 2213 : rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline);
651 : }
652 :
653 0 : struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
654 : {
655 2209 : struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
656 :
657 2209 : if (!left)
658 : return NULL;
659 :
660 2209 : return __node_2_se(left);
661 : }
662 :
663 : static struct sched_entity *__pick_next_entity(struct sched_entity *se)
664 : {
665 0 : struct rb_node *next = rb_next(&se->run_node);
666 :
667 0 : if (!next)
668 : return NULL;
669 :
670 0 : return __node_2_se(next);
671 : }
672 :
673 : #ifdef CONFIG_SCHED_DEBUG
674 0 : struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
675 : {
676 0 : struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);
677 :
678 0 : if (!last)
679 : return NULL;
680 :
681 0 : return __node_2_se(last);
682 : }
683 :
684 : /**************************************************************
685 : * Scheduling class statistics methods:
686 : */
687 :
688 0 : int sched_update_scaling(void)
689 : {
690 0 : unsigned int factor = get_update_sysctl_factor();
691 :
692 0 : sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
693 : sysctl_sched_min_granularity);
694 :
695 : #define WRT_SYSCTL(name) \
696 : (normalized_sysctl_##name = sysctl_##name / (factor))
697 0 : WRT_SYSCTL(sched_min_granularity);
698 0 : WRT_SYSCTL(sched_latency);
699 0 : WRT_SYSCTL(sched_wakeup_granularity);
700 : #undef WRT_SYSCTL
701 :
702 0 : return 0;
703 : }
704 : #endif
705 :
706 : /*
707 : * delta /= w
708 : */
709 : static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
710 : {
711 3809 : if (unlikely(se->load.weight != NICE_0_LOAD))
712 0 : delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
713 :
714 : return delta;
715 : }
716 :
717 : /*
718 : * The idea is to set a period in which each task runs once.
719 : *
720 : * When there are too many tasks (sched_nr_latency) we have to stretch
721 : * this period because otherwise the slices get too small.
722 : *
723 : * p = (nr <= nl) ? l : l*nr/nl
724 : */
725 : static u64 __sched_period(unsigned long nr_running)
726 : {
727 465 : if (unlikely(nr_running > sched_nr_latency))
728 0 : return nr_running * sysctl_sched_min_granularity;
729 : else
730 465 : return sysctl_sched_latency;
731 : }
732 :
733 : static bool sched_idle_cfs_rq(struct cfs_rq *cfs_rq);
734 :
735 : /*
736 : * We calculate the wall-time slice from the period by taking a part
737 : * proportional to the weight.
738 : *
739 : * s = p*P[w/rw]
740 : */
741 465 : static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
742 : {
743 465 : unsigned int nr_running = cfs_rq->nr_running;
744 465 : struct sched_entity *init_se = se;
745 : unsigned int min_gran;
746 : u64 slice;
747 :
748 465 : if (sched_feat(ALT_PERIOD))
749 465 : nr_running = rq_of(cfs_rq)->cfs.h_nr_running;
750 :
751 465 : slice = __sched_period(nr_running + !se->on_rq);
752 :
753 465 : for_each_sched_entity(se) {
754 : struct load_weight *load;
755 : struct load_weight lw;
756 : struct cfs_rq *qcfs_rq;
757 :
758 930 : qcfs_rq = cfs_rq_of(se);
759 465 : load = &qcfs_rq->load;
760 :
761 465 : if (unlikely(!se->on_rq)) {
762 340 : lw = qcfs_rq->load;
763 :
764 680 : update_load_add(&lw, se->load.weight);
765 340 : load = &lw;
766 : }
767 465 : slice = __calc_delta(slice, se->load.weight, load);
768 : }
769 :
770 465 : if (sched_feat(BASE_SLICE)) {
771 465 : if (se_is_idle(init_se) && !sched_idle_cfs_rq(cfs_rq))
772 : min_gran = sysctl_sched_idle_min_granularity;
773 : else
774 465 : min_gran = sysctl_sched_min_granularity;
775 :
776 465 : slice = max_t(u64, slice, min_gran);
777 : }
778 :
779 465 : return slice;
780 : }
781 :
782 : /*
783 : * We calculate the vruntime slice of a to-be-inserted task.
784 : *
785 : * vs = s/w
786 : */
787 340 : static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
788 : {
789 680 : return calc_delta_fair(sched_slice(cfs_rq, se), se);
790 : }
791 :
792 : #include "pelt.h"
793 : #ifdef CONFIG_SMP
794 :
795 : static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
796 : static unsigned long task_h_load(struct task_struct *p);
797 : static unsigned long capacity_of(int cpu);
798 :
799 : /* Give new sched_entity start runnable values to heavy its load in infant time */
800 : void init_entity_runnable_average(struct sched_entity *se)
801 : {
802 : struct sched_avg *sa = &se->avg;
803 :
804 : memset(sa, 0, sizeof(*sa));
805 :
806 : /*
807 : * Tasks are initialized with full load to be seen as heavy tasks until
808 : * they get a chance to stabilize to their real load level.
809 : * Group entities are initialized with zero load to reflect the fact that
810 : * nothing has been attached to the task group yet.
811 : */
812 : if (entity_is_task(se))
813 : sa->load_avg = scale_load_down(se->load.weight);
814 :
815 : /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
816 : }
817 :
818 : /*
819 : * With new tasks being created, their initial util_avgs are extrapolated
820 : * based on the cfs_rq's current util_avg:
821 : *
822 : * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight
823 : *
824 : * However, in many cases, the above util_avg does not give a desired
825 : * value. Moreover, the sum of the util_avgs may be divergent, such
826 : * as when the series is a harmonic series.
827 : *
828 : * To solve this problem, we also cap the util_avg of successive tasks to
829 : * only 1/2 of the left utilization budget:
830 : *
831 : * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n
832 : *
833 : * where n denotes the nth task and cpu_scale the CPU capacity.
834 : *
835 : * For example, for a CPU with 1024 of capacity, a simplest series from
836 : * the beginning would be like:
837 : *
838 : * task util_avg: 512, 256, 128, 64, 32, 16, 8, ...
839 : * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ...
840 : *
841 : * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap)
842 : * if util_avg > util_avg_cap.
843 : */
844 : void post_init_entity_util_avg(struct task_struct *p)
845 : {
846 : struct sched_entity *se = &p->se;
847 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
848 : struct sched_avg *sa = &se->avg;
849 : long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq)));
850 : long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2;
851 :
852 : if (p->sched_class != &fair_sched_class) {
853 : /*
854 : * For !fair tasks do:
855 : *
856 : update_cfs_rq_load_avg(now, cfs_rq);
857 : attach_entity_load_avg(cfs_rq, se);
858 : switched_from_fair(rq, p);
859 : *
860 : * such that the next switched_to_fair() has the
861 : * expected state.
862 : */
863 : se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq);
864 : return;
865 : }
866 :
867 : if (cap > 0) {
868 : if (cfs_rq->avg.util_avg != 0) {
869 : sa->util_avg = cfs_rq->avg.util_avg * se->load.weight;
870 : sa->util_avg /= (cfs_rq->avg.load_avg + 1);
871 :
872 : if (sa->util_avg > cap)
873 : sa->util_avg = cap;
874 : } else {
875 : sa->util_avg = cap;
876 : }
877 : }
878 :
879 : sa->runnable_avg = sa->util_avg;
880 : }
881 :
882 : #else /* !CONFIG_SMP */
883 340 : void init_entity_runnable_average(struct sched_entity *se)
884 : {
885 340 : }
886 340 : void post_init_entity_util_avg(struct task_struct *p)
887 : {
888 340 : }
889 : static void update_tg_load_avg(struct cfs_rq *cfs_rq)
890 : {
891 : }
892 : #endif /* CONFIG_SMP */
893 :
894 : /*
895 : * Update the current task's runtime statistics.
896 : */
897 9180 : static void update_curr(struct cfs_rq *cfs_rq)
898 : {
899 9180 : struct sched_entity *curr = cfs_rq->curr;
900 18360 : u64 now = rq_clock_task(rq_of(cfs_rq));
901 : u64 delta_exec;
902 :
903 9180 : if (unlikely(!curr))
904 : return;
905 :
906 9173 : delta_exec = now - curr->exec_start;
907 9173 : if (unlikely((s64)delta_exec <= 0))
908 : return;
909 :
910 2719 : curr->exec_start = now;
911 :
912 : if (schedstat_enabled()) {
913 : struct sched_statistics *stats;
914 :
915 : stats = __schedstats_from_se(curr);
916 : __schedstat_set(stats->exec_max,
917 : max(delta_exec, stats->exec_max));
918 : }
919 :
920 2719 : curr->sum_exec_runtime += delta_exec;
921 : schedstat_add(cfs_rq->exec_clock, delta_exec);
922 :
923 2719 : curr->vruntime += calc_delta_fair(delta_exec, curr);
924 2719 : update_min_vruntime(cfs_rq);
925 :
926 : if (entity_is_task(curr)) {
927 2719 : struct task_struct *curtask = task_of(curr);
928 :
929 2719 : trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
930 2719 : cgroup_account_cputime(curtask, delta_exec);
931 : account_group_exec_runtime(curtask, delta_exec);
932 : }
933 :
934 : account_cfs_rq_runtime(cfs_rq, delta_exec);
935 : }
936 :
937 0 : static void update_curr_fair(struct rq *rq)
938 : {
939 0 : update_curr(cfs_rq_of(&rq->curr->se));
940 0 : }
941 :
942 : static inline void
943 : update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
944 : {
945 : struct sched_statistics *stats;
946 82 : struct task_struct *p = NULL;
947 :
948 : if (!schedstat_enabled())
949 : return;
950 :
951 : stats = __schedstats_from_se(se);
952 :
953 : if (entity_is_task(se))
954 : p = task_of(se);
955 :
956 : __update_stats_wait_start(rq_of(cfs_rq), p, stats);
957 : }
958 :
959 : static inline void
960 : update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
961 : {
962 : struct sched_statistics *stats;
963 2213 : struct task_struct *p = NULL;
964 :
965 : if (!schedstat_enabled())
966 : return;
967 :
968 : stats = __schedstats_from_se(se);
969 :
970 : /*
971 : * When the sched_schedstat changes from 0 to 1, some sched se
972 : * maybe already in the runqueue, the se->statistics.wait_start
973 : * will be 0.So it will let the delta wrong. We need to avoid this
974 : * scenario.
975 : */
976 : if (unlikely(!schedstat_val(stats->wait_start)))
977 : return;
978 :
979 : if (entity_is_task(se))
980 : p = task_of(se);
981 :
982 : __update_stats_wait_end(rq_of(cfs_rq), p, stats);
983 : }
984 :
985 : static inline void
986 : update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
987 : {
988 : struct sched_statistics *stats;
989 : struct task_struct *tsk = NULL;
990 :
991 : if (!schedstat_enabled())
992 : return;
993 :
994 : stats = __schedstats_from_se(se);
995 :
996 : if (entity_is_task(se))
997 : tsk = task_of(se);
998 :
999 : __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats);
1000 : }
1001 :
1002 : /*
1003 : * Task is being enqueued - update stats:
1004 : */
1005 : static inline void
1006 : update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1007 : {
1008 : if (!schedstat_enabled())
1009 : return;
1010 :
1011 : /*
1012 : * Are we enqueueing a waiting task? (for current tasks
1013 : * a dequeue/enqueue event is a NOP)
1014 : */
1015 : if (se != cfs_rq->curr)
1016 : update_stats_wait_start_fair(cfs_rq, se);
1017 :
1018 : if (flags & ENQUEUE_WAKEUP)
1019 : update_stats_enqueue_sleeper_fair(cfs_rq, se);
1020 : }
1021 :
1022 : static inline void
1023 : update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1024 : {
1025 :
1026 : if (!schedstat_enabled())
1027 : return;
1028 :
1029 : /*
1030 : * Mark the end of the wait period if dequeueing a
1031 : * waiting task:
1032 : */
1033 : if (se != cfs_rq->curr)
1034 : update_stats_wait_end_fair(cfs_rq, se);
1035 :
1036 : if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
1037 : struct task_struct *tsk = task_of(se);
1038 : unsigned int state;
1039 :
1040 : /* XXX racy against TTWU */
1041 : state = READ_ONCE(tsk->__state);
1042 : if (state & TASK_INTERRUPTIBLE)
1043 : __schedstat_set(tsk->stats.sleep_start,
1044 : rq_clock(rq_of(cfs_rq)));
1045 : if (state & TASK_UNINTERRUPTIBLE)
1046 : __schedstat_set(tsk->stats.block_start,
1047 : rq_clock(rq_of(cfs_rq)));
1048 : }
1049 : }
1050 :
1051 : /*
1052 : * We are picking a new current task - update its stats:
1053 : */
1054 : static inline void
1055 : update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
1056 : {
1057 : /*
1058 : * We are starting a new run period:
1059 : */
1060 4426 : se->exec_start = rq_clock_task(rq_of(cfs_rq));
1061 : }
1062 :
1063 : /**************************************************
1064 : * Scheduling class queueing methods:
1065 : */
1066 :
1067 : #ifdef CONFIG_NUMA
1068 : #define NUMA_IMBALANCE_MIN 2
1069 :
1070 : static inline long
1071 : adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr)
1072 : {
1073 : /*
1074 : * Allow a NUMA imbalance if busy CPUs is less than the maximum
1075 : * threshold. Above this threshold, individual tasks may be contending
1076 : * for both memory bandwidth and any shared HT resources. This is an
1077 : * approximation as the number of running tasks may not be related to
1078 : * the number of busy CPUs due to sched_setaffinity.
1079 : */
1080 : if (dst_running > imb_numa_nr)
1081 : return imbalance;
1082 :
1083 : /*
1084 : * Allow a small imbalance based on a simple pair of communicating
1085 : * tasks that remain local when the destination is lightly loaded.
1086 : */
1087 : if (imbalance <= NUMA_IMBALANCE_MIN)
1088 : return 0;
1089 :
1090 : return imbalance;
1091 : }
1092 : #endif /* CONFIG_NUMA */
1093 :
1094 : #ifdef CONFIG_NUMA_BALANCING
1095 : /*
1096 : * Approximate time to scan a full NUMA task in ms. The task scan period is
1097 : * calculated based on the tasks virtual memory size and
1098 : * numa_balancing_scan_size.
1099 : */
1100 : unsigned int sysctl_numa_balancing_scan_period_min = 1000;
1101 : unsigned int sysctl_numa_balancing_scan_period_max = 60000;
1102 :
1103 : /* Portion of address space to scan in MB */
1104 : unsigned int sysctl_numa_balancing_scan_size = 256;
1105 :
1106 : /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
1107 : unsigned int sysctl_numa_balancing_scan_delay = 1000;
1108 :
1109 : /* The page with hint page fault latency < threshold in ms is considered hot */
1110 : unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC;
1111 :
1112 : struct numa_group {
1113 : refcount_t refcount;
1114 :
1115 : spinlock_t lock; /* nr_tasks, tasks */
1116 : int nr_tasks;
1117 : pid_t gid;
1118 : int active_nodes;
1119 :
1120 : struct rcu_head rcu;
1121 : unsigned long total_faults;
1122 : unsigned long max_faults_cpu;
1123 : /*
1124 : * faults[] array is split into two regions: faults_mem and faults_cpu.
1125 : *
1126 : * Faults_cpu is used to decide whether memory should move
1127 : * towards the CPU. As a consequence, these stats are weighted
1128 : * more by CPU use than by memory faults.
1129 : */
1130 : unsigned long faults[];
1131 : };
1132 :
1133 : /*
1134 : * For functions that can be called in multiple contexts that permit reading
1135 : * ->numa_group (see struct task_struct for locking rules).
1136 : */
1137 : static struct numa_group *deref_task_numa_group(struct task_struct *p)
1138 : {
1139 : return rcu_dereference_check(p->numa_group, p == current ||
1140 : (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu)));
1141 : }
1142 :
1143 : static struct numa_group *deref_curr_numa_group(struct task_struct *p)
1144 : {
1145 : return rcu_dereference_protected(p->numa_group, p == current);
1146 : }
1147 :
1148 : static inline unsigned long group_faults_priv(struct numa_group *ng);
1149 : static inline unsigned long group_faults_shared(struct numa_group *ng);
1150 :
1151 : static unsigned int task_nr_scan_windows(struct task_struct *p)
1152 : {
1153 : unsigned long rss = 0;
1154 : unsigned long nr_scan_pages;
1155 :
1156 : /*
1157 : * Calculations based on RSS as non-present and empty pages are skipped
1158 : * by the PTE scanner and NUMA hinting faults should be trapped based
1159 : * on resident pages
1160 : */
1161 : nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
1162 : rss = get_mm_rss(p->mm);
1163 : if (!rss)
1164 : rss = nr_scan_pages;
1165 :
1166 : rss = round_up(rss, nr_scan_pages);
1167 : return rss / nr_scan_pages;
1168 : }
1169 :
1170 : /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
1171 : #define MAX_SCAN_WINDOW 2560
1172 :
1173 : static unsigned int task_scan_min(struct task_struct *p)
1174 : {
1175 : unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
1176 : unsigned int scan, floor;
1177 : unsigned int windows = 1;
1178 :
1179 : if (scan_size < MAX_SCAN_WINDOW)
1180 : windows = MAX_SCAN_WINDOW / scan_size;
1181 : floor = 1000 / windows;
1182 :
1183 : scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
1184 : return max_t(unsigned int, floor, scan);
1185 : }
1186 :
1187 : static unsigned int task_scan_start(struct task_struct *p)
1188 : {
1189 : unsigned long smin = task_scan_min(p);
1190 : unsigned long period = smin;
1191 : struct numa_group *ng;
1192 :
1193 : /* Scale the maximum scan period with the amount of shared memory. */
1194 : rcu_read_lock();
1195 : ng = rcu_dereference(p->numa_group);
1196 : if (ng) {
1197 : unsigned long shared = group_faults_shared(ng);
1198 : unsigned long private = group_faults_priv(ng);
1199 :
1200 : period *= refcount_read(&ng->refcount);
1201 : period *= shared + 1;
1202 : period /= private + shared + 1;
1203 : }
1204 : rcu_read_unlock();
1205 :
1206 : return max(smin, period);
1207 : }
1208 :
1209 : static unsigned int task_scan_max(struct task_struct *p)
1210 : {
1211 : unsigned long smin = task_scan_min(p);
1212 : unsigned long smax;
1213 : struct numa_group *ng;
1214 :
1215 : /* Watch for min being lower than max due to floor calculations */
1216 : smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
1217 :
1218 : /* Scale the maximum scan period with the amount of shared memory. */
1219 : ng = deref_curr_numa_group(p);
1220 : if (ng) {
1221 : unsigned long shared = group_faults_shared(ng);
1222 : unsigned long private = group_faults_priv(ng);
1223 : unsigned long period = smax;
1224 :
1225 : period *= refcount_read(&ng->refcount);
1226 : period *= shared + 1;
1227 : period /= private + shared + 1;
1228 :
1229 : smax = max(smax, period);
1230 : }
1231 :
1232 : return max(smin, smax);
1233 : }
1234 :
1235 : static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
1236 : {
1237 : rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE);
1238 : rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
1239 : }
1240 :
1241 : static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
1242 : {
1243 : rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE);
1244 : rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
1245 : }
1246 :
1247 : /* Shared or private faults. */
1248 : #define NR_NUMA_HINT_FAULT_TYPES 2
1249 :
1250 : /* Memory and CPU locality */
1251 : #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
1252 :
1253 : /* Averaged statistics, and temporary buffers. */
1254 : #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
1255 :
1256 : pid_t task_numa_group_id(struct task_struct *p)
1257 : {
1258 : struct numa_group *ng;
1259 : pid_t gid = 0;
1260 :
1261 : rcu_read_lock();
1262 : ng = rcu_dereference(p->numa_group);
1263 : if (ng)
1264 : gid = ng->gid;
1265 : rcu_read_unlock();
1266 :
1267 : return gid;
1268 : }
1269 :
1270 : /*
1271 : * The averaged statistics, shared & private, memory & CPU,
1272 : * occupy the first half of the array. The second half of the
1273 : * array is for current counters, which are averaged into the
1274 : * first set by task_numa_placement.
1275 : */
1276 : static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
1277 : {
1278 : return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
1279 : }
1280 :
1281 : static inline unsigned long task_faults(struct task_struct *p, int nid)
1282 : {
1283 : if (!p->numa_faults)
1284 : return 0;
1285 :
1286 : return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1287 : p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];
1288 : }
1289 :
1290 : static inline unsigned long group_faults(struct task_struct *p, int nid)
1291 : {
1292 : struct numa_group *ng = deref_task_numa_group(p);
1293 :
1294 : if (!ng)
1295 : return 0;
1296 :
1297 : return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1298 : ng->faults[task_faults_idx(NUMA_MEM, nid, 1)];
1299 : }
1300 :
1301 : static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
1302 : {
1303 : return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] +
1304 : group->faults[task_faults_idx(NUMA_CPU, nid, 1)];
1305 : }
1306 :
1307 : static inline unsigned long group_faults_priv(struct numa_group *ng)
1308 : {
1309 : unsigned long faults = 0;
1310 : int node;
1311 :
1312 : for_each_online_node(node) {
1313 : faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
1314 : }
1315 :
1316 : return faults;
1317 : }
1318 :
1319 : static inline unsigned long group_faults_shared(struct numa_group *ng)
1320 : {
1321 : unsigned long faults = 0;
1322 : int node;
1323 :
1324 : for_each_online_node(node) {
1325 : faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
1326 : }
1327 :
1328 : return faults;
1329 : }
1330 :
1331 : /*
1332 : * A node triggering more than 1/3 as many NUMA faults as the maximum is
1333 : * considered part of a numa group's pseudo-interleaving set. Migrations
1334 : * between these nodes are slowed down, to allow things to settle down.
1335 : */
1336 : #define ACTIVE_NODE_FRACTION 3
1337 :
1338 : static bool numa_is_active_node(int nid, struct numa_group *ng)
1339 : {
1340 : return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
1341 : }
1342 :
1343 : /* Handle placement on systems where not all nodes are directly connected. */
1344 : static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
1345 : int lim_dist, bool task)
1346 : {
1347 : unsigned long score = 0;
1348 : int node, max_dist;
1349 :
1350 : /*
1351 : * All nodes are directly connected, and the same distance
1352 : * from each other. No need for fancy placement algorithms.
1353 : */
1354 : if (sched_numa_topology_type == NUMA_DIRECT)
1355 : return 0;
1356 :
1357 : /* sched_max_numa_distance may be changed in parallel. */
1358 : max_dist = READ_ONCE(sched_max_numa_distance);
1359 : /*
1360 : * This code is called for each node, introducing N^2 complexity,
1361 : * which should be ok given the number of nodes rarely exceeds 8.
1362 : */
1363 : for_each_online_node(node) {
1364 : unsigned long faults;
1365 : int dist = node_distance(nid, node);
1366 :
1367 : /*
1368 : * The furthest away nodes in the system are not interesting
1369 : * for placement; nid was already counted.
1370 : */
1371 : if (dist >= max_dist || node == nid)
1372 : continue;
1373 :
1374 : /*
1375 : * On systems with a backplane NUMA topology, compare groups
1376 : * of nodes, and move tasks towards the group with the most
1377 : * memory accesses. When comparing two nodes at distance
1378 : * "hoplimit", only nodes closer by than "hoplimit" are part
1379 : * of each group. Skip other nodes.
1380 : */
1381 : if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist)
1382 : continue;
1383 :
1384 : /* Add up the faults from nearby nodes. */
1385 : if (task)
1386 : faults = task_faults(p, node);
1387 : else
1388 : faults = group_faults(p, node);
1389 :
1390 : /*
1391 : * On systems with a glueless mesh NUMA topology, there are
1392 : * no fixed "groups of nodes". Instead, nodes that are not
1393 : * directly connected bounce traffic through intermediate
1394 : * nodes; a numa_group can occupy any set of nodes.
1395 : * The further away a node is, the less the faults count.
1396 : * This seems to result in good task placement.
1397 : */
1398 : if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
1399 : faults *= (max_dist - dist);
1400 : faults /= (max_dist - LOCAL_DISTANCE);
1401 : }
1402 :
1403 : score += faults;
1404 : }
1405 :
1406 : return score;
1407 : }
1408 :
1409 : /*
1410 : * These return the fraction of accesses done by a particular task, or
1411 : * task group, on a particular numa node. The group weight is given a
1412 : * larger multiplier, in order to group tasks together that are almost
1413 : * evenly spread out between numa nodes.
1414 : */
1415 : static inline unsigned long task_weight(struct task_struct *p, int nid,
1416 : int dist)
1417 : {
1418 : unsigned long faults, total_faults;
1419 :
1420 : if (!p->numa_faults)
1421 : return 0;
1422 :
1423 : total_faults = p->total_numa_faults;
1424 :
1425 : if (!total_faults)
1426 : return 0;
1427 :
1428 : faults = task_faults(p, nid);
1429 : faults += score_nearby_nodes(p, nid, dist, true);
1430 :
1431 : return 1000 * faults / total_faults;
1432 : }
1433 :
1434 : static inline unsigned long group_weight(struct task_struct *p, int nid,
1435 : int dist)
1436 : {
1437 : struct numa_group *ng = deref_task_numa_group(p);
1438 : unsigned long faults, total_faults;
1439 :
1440 : if (!ng)
1441 : return 0;
1442 :
1443 : total_faults = ng->total_faults;
1444 :
1445 : if (!total_faults)
1446 : return 0;
1447 :
1448 : faults = group_faults(p, nid);
1449 : faults += score_nearby_nodes(p, nid, dist, false);
1450 :
1451 : return 1000 * faults / total_faults;
1452 : }
1453 :
1454 : /*
1455 : * If memory tiering mode is enabled, cpupid of slow memory page is
1456 : * used to record scan time instead of CPU and PID. When tiering mode
1457 : * is disabled at run time, the scan time (in cpupid) will be
1458 : * interpreted as CPU and PID. So CPU needs to be checked to avoid to
1459 : * access out of array bound.
1460 : */
1461 : static inline bool cpupid_valid(int cpupid)
1462 : {
1463 : return cpupid_to_cpu(cpupid) < nr_cpu_ids;
1464 : }
1465 :
1466 : /*
1467 : * For memory tiering mode, if there are enough free pages (more than
1468 : * enough watermark defined here) in fast memory node, to take full
1469 : * advantage of fast memory capacity, all recently accessed slow
1470 : * memory pages will be migrated to fast memory node without
1471 : * considering hot threshold.
1472 : */
1473 : static bool pgdat_free_space_enough(struct pglist_data *pgdat)
1474 : {
1475 : int z;
1476 : unsigned long enough_wmark;
1477 :
1478 : enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT,
1479 : pgdat->node_present_pages >> 4);
1480 : for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1481 : struct zone *zone = pgdat->node_zones + z;
1482 :
1483 : if (!populated_zone(zone))
1484 : continue;
1485 :
1486 : if (zone_watermark_ok(zone, 0,
1487 : wmark_pages(zone, WMARK_PROMO) + enough_wmark,
1488 : ZONE_MOVABLE, 0))
1489 : return true;
1490 : }
1491 : return false;
1492 : }
1493 :
1494 : /*
1495 : * For memory tiering mode, when page tables are scanned, the scan
1496 : * time will be recorded in struct page in addition to make page
1497 : * PROT_NONE for slow memory page. So when the page is accessed, in
1498 : * hint page fault handler, the hint page fault latency is calculated
1499 : * via,
1500 : *
1501 : * hint page fault latency = hint page fault time - scan time
1502 : *
1503 : * The smaller the hint page fault latency, the higher the possibility
1504 : * for the page to be hot.
1505 : */
1506 : static int numa_hint_fault_latency(struct page *page)
1507 : {
1508 : int last_time, time;
1509 :
1510 : time = jiffies_to_msecs(jiffies);
1511 : last_time = xchg_page_access_time(page, time);
1512 :
1513 : return (time - last_time) & PAGE_ACCESS_TIME_MASK;
1514 : }
1515 :
1516 : /*
1517 : * For memory tiering mode, too high promotion/demotion throughput may
1518 : * hurt application latency. So we provide a mechanism to rate limit
1519 : * the number of pages that are tried to be promoted.
1520 : */
1521 : static bool numa_promotion_rate_limit(struct pglist_data *pgdat,
1522 : unsigned long rate_limit, int nr)
1523 : {
1524 : unsigned long nr_cand;
1525 : unsigned int now, start;
1526 :
1527 : now = jiffies_to_msecs(jiffies);
1528 : mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr);
1529 : nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
1530 : start = pgdat->nbp_rl_start;
1531 : if (now - start > MSEC_PER_SEC &&
1532 : cmpxchg(&pgdat->nbp_rl_start, start, now) == start)
1533 : pgdat->nbp_rl_nr_cand = nr_cand;
1534 : if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit)
1535 : return true;
1536 : return false;
1537 : }
1538 :
1539 : #define NUMA_MIGRATION_ADJUST_STEPS 16
1540 :
1541 : static void numa_promotion_adjust_threshold(struct pglist_data *pgdat,
1542 : unsigned long rate_limit,
1543 : unsigned int ref_th)
1544 : {
1545 : unsigned int now, start, th_period, unit_th, th;
1546 : unsigned long nr_cand, ref_cand, diff_cand;
1547 :
1548 : now = jiffies_to_msecs(jiffies);
1549 : th_period = sysctl_numa_balancing_scan_period_max;
1550 : start = pgdat->nbp_th_start;
1551 : if (now - start > th_period &&
1552 : cmpxchg(&pgdat->nbp_th_start, start, now) == start) {
1553 : ref_cand = rate_limit *
1554 : sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC;
1555 : nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
1556 : diff_cand = nr_cand - pgdat->nbp_th_nr_cand;
1557 : unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS;
1558 : th = pgdat->nbp_threshold ? : ref_th;
1559 : if (diff_cand > ref_cand * 11 / 10)
1560 : th = max(th - unit_th, unit_th);
1561 : else if (diff_cand < ref_cand * 9 / 10)
1562 : th = min(th + unit_th, ref_th * 2);
1563 : pgdat->nbp_th_nr_cand = nr_cand;
1564 : pgdat->nbp_threshold = th;
1565 : }
1566 : }
1567 :
1568 : bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
1569 : int src_nid, int dst_cpu)
1570 : {
1571 : struct numa_group *ng = deref_curr_numa_group(p);
1572 : int dst_nid = cpu_to_node(dst_cpu);
1573 : int last_cpupid, this_cpupid;
1574 :
1575 : /*
1576 : * The pages in slow memory node should be migrated according
1577 : * to hot/cold instead of private/shared.
1578 : */
1579 : if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
1580 : !node_is_toptier(src_nid)) {
1581 : struct pglist_data *pgdat;
1582 : unsigned long rate_limit;
1583 : unsigned int latency, th, def_th;
1584 :
1585 : pgdat = NODE_DATA(dst_nid);
1586 : if (pgdat_free_space_enough(pgdat)) {
1587 : /* workload changed, reset hot threshold */
1588 : pgdat->nbp_threshold = 0;
1589 : return true;
1590 : }
1591 :
1592 : def_th = sysctl_numa_balancing_hot_threshold;
1593 : rate_limit = sysctl_numa_balancing_promote_rate_limit << \
1594 : (20 - PAGE_SHIFT);
1595 : numa_promotion_adjust_threshold(pgdat, rate_limit, def_th);
1596 :
1597 : th = pgdat->nbp_threshold ? : def_th;
1598 : latency = numa_hint_fault_latency(page);
1599 : if (latency >= th)
1600 : return false;
1601 :
1602 : return !numa_promotion_rate_limit(pgdat, rate_limit,
1603 : thp_nr_pages(page));
1604 : }
1605 :
1606 : this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
1607 : last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
1608 :
1609 : if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
1610 : !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid))
1611 : return false;
1612 :
1613 : /*
1614 : * Allow first faults or private faults to migrate immediately early in
1615 : * the lifetime of a task. The magic number 4 is based on waiting for
1616 : * two full passes of the "multi-stage node selection" test that is
1617 : * executed below.
1618 : */
1619 : if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) &&
1620 : (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid)))
1621 : return true;
1622 :
1623 : /*
1624 : * Multi-stage node selection is used in conjunction with a periodic
1625 : * migration fault to build a temporal task<->page relation. By using
1626 : * a two-stage filter we remove short/unlikely relations.
1627 : *
1628 : * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
1629 : * a task's usage of a particular page (n_p) per total usage of this
1630 : * page (n_t) (in a given time-span) to a probability.
1631 : *
1632 : * Our periodic faults will sample this probability and getting the
1633 : * same result twice in a row, given these samples are fully
1634 : * independent, is then given by P(n)^2, provided our sample period
1635 : * is sufficiently short compared to the usage pattern.
1636 : *
1637 : * This quadric squishes small probabilities, making it less likely we
1638 : * act on an unlikely task<->page relation.
1639 : */
1640 : if (!cpupid_pid_unset(last_cpupid) &&
1641 : cpupid_to_nid(last_cpupid) != dst_nid)
1642 : return false;
1643 :
1644 : /* Always allow migrate on private faults */
1645 : if (cpupid_match_pid(p, last_cpupid))
1646 : return true;
1647 :
1648 : /* A shared fault, but p->numa_group has not been set up yet. */
1649 : if (!ng)
1650 : return true;
1651 :
1652 : /*
1653 : * Destination node is much more heavily used than the source
1654 : * node? Allow migration.
1655 : */
1656 : if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) *
1657 : ACTIVE_NODE_FRACTION)
1658 : return true;
1659 :
1660 : /*
1661 : * Distribute memory according to CPU & memory use on each node,
1662 : * with 3/4 hysteresis to avoid unnecessary memory migrations:
1663 : *
1664 : * faults_cpu(dst) 3 faults_cpu(src)
1665 : * --------------- * - > ---------------
1666 : * faults_mem(dst) 4 faults_mem(src)
1667 : */
1668 : return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 >
1669 : group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4;
1670 : }
1671 :
1672 : /*
1673 : * 'numa_type' describes the node at the moment of load balancing.
1674 : */
1675 : enum numa_type {
1676 : /* The node has spare capacity that can be used to run more tasks. */
1677 : node_has_spare = 0,
1678 : /*
1679 : * The node is fully used and the tasks don't compete for more CPU
1680 : * cycles. Nevertheless, some tasks might wait before running.
1681 : */
1682 : node_fully_busy,
1683 : /*
1684 : * The node is overloaded and can't provide expected CPU cycles to all
1685 : * tasks.
1686 : */
1687 : node_overloaded
1688 : };
1689 :
1690 : /* Cached statistics for all CPUs within a node */
1691 : struct numa_stats {
1692 : unsigned long load;
1693 : unsigned long runnable;
1694 : unsigned long util;
1695 : /* Total compute capacity of CPUs on a node */
1696 : unsigned long compute_capacity;
1697 : unsigned int nr_running;
1698 : unsigned int weight;
1699 : enum numa_type node_type;
1700 : int idle_cpu;
1701 : };
1702 :
1703 : static inline bool is_core_idle(int cpu)
1704 : {
1705 : #ifdef CONFIG_SCHED_SMT
1706 : int sibling;
1707 :
1708 : for_each_cpu(sibling, cpu_smt_mask(cpu)) {
1709 : if (cpu == sibling)
1710 : continue;
1711 :
1712 : if (!idle_cpu(sibling))
1713 : return false;
1714 : }
1715 : #endif
1716 :
1717 : return true;
1718 : }
1719 :
1720 : struct task_numa_env {
1721 : struct task_struct *p;
1722 :
1723 : int src_cpu, src_nid;
1724 : int dst_cpu, dst_nid;
1725 : int imb_numa_nr;
1726 :
1727 : struct numa_stats src_stats, dst_stats;
1728 :
1729 : int imbalance_pct;
1730 : int dist;
1731 :
1732 : struct task_struct *best_task;
1733 : long best_imp;
1734 : int best_cpu;
1735 : };
1736 :
1737 : static unsigned long cpu_load(struct rq *rq);
1738 : static unsigned long cpu_runnable(struct rq *rq);
1739 :
1740 : static inline enum
1741 : numa_type numa_classify(unsigned int imbalance_pct,
1742 : struct numa_stats *ns)
1743 : {
1744 : if ((ns->nr_running > ns->weight) &&
1745 : (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) ||
1746 : ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100))))
1747 : return node_overloaded;
1748 :
1749 : if ((ns->nr_running < ns->weight) ||
1750 : (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) &&
1751 : ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100))))
1752 : return node_has_spare;
1753 :
1754 : return node_fully_busy;
1755 : }
1756 :
1757 : #ifdef CONFIG_SCHED_SMT
1758 : /* Forward declarations of select_idle_sibling helpers */
1759 : static inline bool test_idle_cores(int cpu);
1760 : static inline int numa_idle_core(int idle_core, int cpu)
1761 : {
1762 : if (!static_branch_likely(&sched_smt_present) ||
1763 : idle_core >= 0 || !test_idle_cores(cpu))
1764 : return idle_core;
1765 :
1766 : /*
1767 : * Prefer cores instead of packing HT siblings
1768 : * and triggering future load balancing.
1769 : */
1770 : if (is_core_idle(cpu))
1771 : idle_core = cpu;
1772 :
1773 : return idle_core;
1774 : }
1775 : #else
1776 : static inline int numa_idle_core(int idle_core, int cpu)
1777 : {
1778 : return idle_core;
1779 : }
1780 : #endif
1781 :
1782 : /*
1783 : * Gather all necessary information to make NUMA balancing placement
1784 : * decisions that are compatible with standard load balancer. This
1785 : * borrows code and logic from update_sg_lb_stats but sharing a
1786 : * common implementation is impractical.
1787 : */
1788 : static void update_numa_stats(struct task_numa_env *env,
1789 : struct numa_stats *ns, int nid,
1790 : bool find_idle)
1791 : {
1792 : int cpu, idle_core = -1;
1793 :
1794 : memset(ns, 0, sizeof(*ns));
1795 : ns->idle_cpu = -1;
1796 :
1797 : rcu_read_lock();
1798 : for_each_cpu(cpu, cpumask_of_node(nid)) {
1799 : struct rq *rq = cpu_rq(cpu);
1800 :
1801 : ns->load += cpu_load(rq);
1802 : ns->runnable += cpu_runnable(rq);
1803 : ns->util += cpu_util_cfs(cpu);
1804 : ns->nr_running += rq->cfs.h_nr_running;
1805 : ns->compute_capacity += capacity_of(cpu);
1806 :
1807 : if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) {
1808 : if (READ_ONCE(rq->numa_migrate_on) ||
1809 : !cpumask_test_cpu(cpu, env->p->cpus_ptr))
1810 : continue;
1811 :
1812 : if (ns->idle_cpu == -1)
1813 : ns->idle_cpu = cpu;
1814 :
1815 : idle_core = numa_idle_core(idle_core, cpu);
1816 : }
1817 : }
1818 : rcu_read_unlock();
1819 :
1820 : ns->weight = cpumask_weight(cpumask_of_node(nid));
1821 :
1822 : ns->node_type = numa_classify(env->imbalance_pct, ns);
1823 :
1824 : if (idle_core >= 0)
1825 : ns->idle_cpu = idle_core;
1826 : }
1827 :
1828 : static void task_numa_assign(struct task_numa_env *env,
1829 : struct task_struct *p, long imp)
1830 : {
1831 : struct rq *rq = cpu_rq(env->dst_cpu);
1832 :
1833 : /* Check if run-queue part of active NUMA balance. */
1834 : if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {
1835 : int cpu;
1836 : int start = env->dst_cpu;
1837 :
1838 : /* Find alternative idle CPU. */
1839 : for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) {
1840 : if (cpu == env->best_cpu || !idle_cpu(cpu) ||
1841 : !cpumask_test_cpu(cpu, env->p->cpus_ptr)) {
1842 : continue;
1843 : }
1844 :
1845 : env->dst_cpu = cpu;
1846 : rq = cpu_rq(env->dst_cpu);
1847 : if (!xchg(&rq->numa_migrate_on, 1))
1848 : goto assign;
1849 : }
1850 :
1851 : /* Failed to find an alternative idle CPU */
1852 : return;
1853 : }
1854 :
1855 : assign:
1856 : /*
1857 : * Clear previous best_cpu/rq numa-migrate flag, since task now
1858 : * found a better CPU to move/swap.
1859 : */
1860 : if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {
1861 : rq = cpu_rq(env->best_cpu);
1862 : WRITE_ONCE(rq->numa_migrate_on, 0);
1863 : }
1864 :
1865 : if (env->best_task)
1866 : put_task_struct(env->best_task);
1867 : if (p)
1868 : get_task_struct(p);
1869 :
1870 : env->best_task = p;
1871 : env->best_imp = imp;
1872 : env->best_cpu = env->dst_cpu;
1873 : }
1874 :
1875 : static bool load_too_imbalanced(long src_load, long dst_load,
1876 : struct task_numa_env *env)
1877 : {
1878 : long imb, old_imb;
1879 : long orig_src_load, orig_dst_load;
1880 : long src_capacity, dst_capacity;
1881 :
1882 : /*
1883 : * The load is corrected for the CPU capacity available on each node.
1884 : *
1885 : * src_load dst_load
1886 : * ------------ vs ---------
1887 : * src_capacity dst_capacity
1888 : */
1889 : src_capacity = env->src_stats.compute_capacity;
1890 : dst_capacity = env->dst_stats.compute_capacity;
1891 :
1892 : imb = abs(dst_load * src_capacity - src_load * dst_capacity);
1893 :
1894 : orig_src_load = env->src_stats.load;
1895 : orig_dst_load = env->dst_stats.load;
1896 :
1897 : old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity);
1898 :
1899 : /* Would this change make things worse? */
1900 : return (imb > old_imb);
1901 : }
1902 :
1903 : /*
1904 : * Maximum NUMA importance can be 1998 (2*999);
1905 : * SMALLIMP @ 30 would be close to 1998/64.
1906 : * Used to deter task migration.
1907 : */
1908 : #define SMALLIMP 30
1909 :
1910 : /*
1911 : * This checks if the overall compute and NUMA accesses of the system would
1912 : * be improved if the source tasks was migrated to the target dst_cpu taking
1913 : * into account that it might be best if task running on the dst_cpu should
1914 : * be exchanged with the source task
1915 : */
1916 : static bool task_numa_compare(struct task_numa_env *env,
1917 : long taskimp, long groupimp, bool maymove)
1918 : {
1919 : struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p);
1920 : struct rq *dst_rq = cpu_rq(env->dst_cpu);
1921 : long imp = p_ng ? groupimp : taskimp;
1922 : struct task_struct *cur;
1923 : long src_load, dst_load;
1924 : int dist = env->dist;
1925 : long moveimp = imp;
1926 : long load;
1927 : bool stopsearch = false;
1928 :
1929 : if (READ_ONCE(dst_rq->numa_migrate_on))
1930 : return false;
1931 :
1932 : rcu_read_lock();
1933 : cur = rcu_dereference(dst_rq->curr);
1934 : if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur)))
1935 : cur = NULL;
1936 :
1937 : /*
1938 : * Because we have preemption enabled we can get migrated around and
1939 : * end try selecting ourselves (current == env->p) as a swap candidate.
1940 : */
1941 : if (cur == env->p) {
1942 : stopsearch = true;
1943 : goto unlock;
1944 : }
1945 :
1946 : if (!cur) {
1947 : if (maymove && moveimp >= env->best_imp)
1948 : goto assign;
1949 : else
1950 : goto unlock;
1951 : }
1952 :
1953 : /* Skip this swap candidate if cannot move to the source cpu. */
1954 : if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))
1955 : goto unlock;
1956 :
1957 : /*
1958 : * Skip this swap candidate if it is not moving to its preferred
1959 : * node and the best task is.
1960 : */
1961 : if (env->best_task &&
1962 : env->best_task->numa_preferred_nid == env->src_nid &&
1963 : cur->numa_preferred_nid != env->src_nid) {
1964 : goto unlock;
1965 : }
1966 :
1967 : /*
1968 : * "imp" is the fault differential for the source task between the
1969 : * source and destination node. Calculate the total differential for
1970 : * the source task and potential destination task. The more negative
1971 : * the value is, the more remote accesses that would be expected to
1972 : * be incurred if the tasks were swapped.
1973 : *
1974 : * If dst and source tasks are in the same NUMA group, or not
1975 : * in any group then look only at task weights.
1976 : */
1977 : cur_ng = rcu_dereference(cur->numa_group);
1978 : if (cur_ng == p_ng) {
1979 : /*
1980 : * Do not swap within a group or between tasks that have
1981 : * no group if there is spare capacity. Swapping does
1982 : * not address the load imbalance and helps one task at
1983 : * the cost of punishing another.
1984 : */
1985 : if (env->dst_stats.node_type == node_has_spare)
1986 : goto unlock;
1987 :
1988 : imp = taskimp + task_weight(cur, env->src_nid, dist) -
1989 : task_weight(cur, env->dst_nid, dist);
1990 : /*
1991 : * Add some hysteresis to prevent swapping the
1992 : * tasks within a group over tiny differences.
1993 : */
1994 : if (cur_ng)
1995 : imp -= imp / 16;
1996 : } else {
1997 : /*
1998 : * Compare the group weights. If a task is all by itself
1999 : * (not part of a group), use the task weight instead.
2000 : */
2001 : if (cur_ng && p_ng)
2002 : imp += group_weight(cur, env->src_nid, dist) -
2003 : group_weight(cur, env->dst_nid, dist);
2004 : else
2005 : imp += task_weight(cur, env->src_nid, dist) -
2006 : task_weight(cur, env->dst_nid, dist);
2007 : }
2008 :
2009 : /* Discourage picking a task already on its preferred node */
2010 : if (cur->numa_preferred_nid == env->dst_nid)
2011 : imp -= imp / 16;
2012 :
2013 : /*
2014 : * Encourage picking a task that moves to its preferred node.
2015 : * This potentially makes imp larger than it's maximum of
2016 : * 1998 (see SMALLIMP and task_weight for why) but in this
2017 : * case, it does not matter.
2018 : */
2019 : if (cur->numa_preferred_nid == env->src_nid)
2020 : imp += imp / 8;
2021 :
2022 : if (maymove && moveimp > imp && moveimp > env->best_imp) {
2023 : imp = moveimp;
2024 : cur = NULL;
2025 : goto assign;
2026 : }
2027 :
2028 : /*
2029 : * Prefer swapping with a task moving to its preferred node over a
2030 : * task that is not.
2031 : */
2032 : if (env->best_task && cur->numa_preferred_nid == env->src_nid &&
2033 : env->best_task->numa_preferred_nid != env->src_nid) {
2034 : goto assign;
2035 : }
2036 :
2037 : /*
2038 : * If the NUMA importance is less than SMALLIMP,
2039 : * task migration might only result in ping pong
2040 : * of tasks and also hurt performance due to cache
2041 : * misses.
2042 : */
2043 : if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)
2044 : goto unlock;
2045 :
2046 : /*
2047 : * In the overloaded case, try and keep the load balanced.
2048 : */
2049 : load = task_h_load(env->p) - task_h_load(cur);
2050 : if (!load)
2051 : goto assign;
2052 :
2053 : dst_load = env->dst_stats.load + load;
2054 : src_load = env->src_stats.load - load;
2055 :
2056 : if (load_too_imbalanced(src_load, dst_load, env))
2057 : goto unlock;
2058 :
2059 : assign:
2060 : /* Evaluate an idle CPU for a task numa move. */
2061 : if (!cur) {
2062 : int cpu = env->dst_stats.idle_cpu;
2063 :
2064 : /* Nothing cached so current CPU went idle since the search. */
2065 : if (cpu < 0)
2066 : cpu = env->dst_cpu;
2067 :
2068 : /*
2069 : * If the CPU is no longer truly idle and the previous best CPU
2070 : * is, keep using it.
2071 : */
2072 : if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
2073 : idle_cpu(env->best_cpu)) {
2074 : cpu = env->best_cpu;
2075 : }
2076 :
2077 : env->dst_cpu = cpu;
2078 : }
2079 :
2080 : task_numa_assign(env, cur, imp);
2081 :
2082 : /*
2083 : * If a move to idle is allowed because there is capacity or load
2084 : * balance improves then stop the search. While a better swap
2085 : * candidate may exist, a search is not free.
2086 : */
2087 : if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu))
2088 : stopsearch = true;
2089 :
2090 : /*
2091 : * If a swap candidate must be identified and the current best task
2092 : * moves its preferred node then stop the search.
2093 : */
2094 : if (!maymove && env->best_task &&
2095 : env->best_task->numa_preferred_nid == env->src_nid) {
2096 : stopsearch = true;
2097 : }
2098 : unlock:
2099 : rcu_read_unlock();
2100 :
2101 : return stopsearch;
2102 : }
2103 :
2104 : static void task_numa_find_cpu(struct task_numa_env *env,
2105 : long taskimp, long groupimp)
2106 : {
2107 : bool maymove = false;
2108 : int cpu;
2109 :
2110 : /*
2111 : * If dst node has spare capacity, then check if there is an
2112 : * imbalance that would be overruled by the load balancer.
2113 : */
2114 : if (env->dst_stats.node_type == node_has_spare) {
2115 : unsigned int imbalance;
2116 : int src_running, dst_running;
2117 :
2118 : /*
2119 : * Would movement cause an imbalance? Note that if src has
2120 : * more running tasks that the imbalance is ignored as the
2121 : * move improves the imbalance from the perspective of the
2122 : * CPU load balancer.
2123 : * */
2124 : src_running = env->src_stats.nr_running - 1;
2125 : dst_running = env->dst_stats.nr_running + 1;
2126 : imbalance = max(0, dst_running - src_running);
2127 : imbalance = adjust_numa_imbalance(imbalance, dst_running,
2128 : env->imb_numa_nr);
2129 :
2130 : /* Use idle CPU if there is no imbalance */
2131 : if (!imbalance) {
2132 : maymove = true;
2133 : if (env->dst_stats.idle_cpu >= 0) {
2134 : env->dst_cpu = env->dst_stats.idle_cpu;
2135 : task_numa_assign(env, NULL, 0);
2136 : return;
2137 : }
2138 : }
2139 : } else {
2140 : long src_load, dst_load, load;
2141 : /*
2142 : * If the improvement from just moving env->p direction is better
2143 : * than swapping tasks around, check if a move is possible.
2144 : */
2145 : load = task_h_load(env->p);
2146 : dst_load = env->dst_stats.load + load;
2147 : src_load = env->src_stats.load - load;
2148 : maymove = !load_too_imbalanced(src_load, dst_load, env);
2149 : }
2150 :
2151 : for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
2152 : /* Skip this CPU if the source task cannot migrate */
2153 : if (!cpumask_test_cpu(cpu, env->p->cpus_ptr))
2154 : continue;
2155 :
2156 : env->dst_cpu = cpu;
2157 : if (task_numa_compare(env, taskimp, groupimp, maymove))
2158 : break;
2159 : }
2160 : }
2161 :
2162 : static int task_numa_migrate(struct task_struct *p)
2163 : {
2164 : struct task_numa_env env = {
2165 : .p = p,
2166 :
2167 : .src_cpu = task_cpu(p),
2168 : .src_nid = task_node(p),
2169 :
2170 : .imbalance_pct = 112,
2171 :
2172 : .best_task = NULL,
2173 : .best_imp = 0,
2174 : .best_cpu = -1,
2175 : };
2176 : unsigned long taskweight, groupweight;
2177 : struct sched_domain *sd;
2178 : long taskimp, groupimp;
2179 : struct numa_group *ng;
2180 : struct rq *best_rq;
2181 : int nid, ret, dist;
2182 :
2183 : /*
2184 : * Pick the lowest SD_NUMA domain, as that would have the smallest
2185 : * imbalance and would be the first to start moving tasks about.
2186 : *
2187 : * And we want to avoid any moving of tasks about, as that would create
2188 : * random movement of tasks -- counter the numa conditions we're trying
2189 : * to satisfy here.
2190 : */
2191 : rcu_read_lock();
2192 : sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
2193 : if (sd) {
2194 : env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
2195 : env.imb_numa_nr = sd->imb_numa_nr;
2196 : }
2197 : rcu_read_unlock();
2198 :
2199 : /*
2200 : * Cpusets can break the scheduler domain tree into smaller
2201 : * balance domains, some of which do not cross NUMA boundaries.
2202 : * Tasks that are "trapped" in such domains cannot be migrated
2203 : * elsewhere, so there is no point in (re)trying.
2204 : */
2205 : if (unlikely(!sd)) {
2206 : sched_setnuma(p, task_node(p));
2207 : return -EINVAL;
2208 : }
2209 :
2210 : env.dst_nid = p->numa_preferred_nid;
2211 : dist = env.dist = node_distance(env.src_nid, env.dst_nid);
2212 : taskweight = task_weight(p, env.src_nid, dist);
2213 : groupweight = group_weight(p, env.src_nid, dist);
2214 : update_numa_stats(&env, &env.src_stats, env.src_nid, false);
2215 : taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
2216 : groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
2217 : update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2218 :
2219 : /* Try to find a spot on the preferred nid. */
2220 : task_numa_find_cpu(&env, taskimp, groupimp);
2221 :
2222 : /*
2223 : * Look at other nodes in these cases:
2224 : * - there is no space available on the preferred_nid
2225 : * - the task is part of a numa_group that is interleaved across
2226 : * multiple NUMA nodes; in order to better consolidate the group,
2227 : * we need to check other locations.
2228 : */
2229 : ng = deref_curr_numa_group(p);
2230 : if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) {
2231 : for_each_node_state(nid, N_CPU) {
2232 : if (nid == env.src_nid || nid == p->numa_preferred_nid)
2233 : continue;
2234 :
2235 : dist = node_distance(env.src_nid, env.dst_nid);
2236 : if (sched_numa_topology_type == NUMA_BACKPLANE &&
2237 : dist != env.dist) {
2238 : taskweight = task_weight(p, env.src_nid, dist);
2239 : groupweight = group_weight(p, env.src_nid, dist);
2240 : }
2241 :
2242 : /* Only consider nodes where both task and groups benefit */
2243 : taskimp = task_weight(p, nid, dist) - taskweight;
2244 : groupimp = group_weight(p, nid, dist) - groupweight;
2245 : if (taskimp < 0 && groupimp < 0)
2246 : continue;
2247 :
2248 : env.dist = dist;
2249 : env.dst_nid = nid;
2250 : update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
2251 : task_numa_find_cpu(&env, taskimp, groupimp);
2252 : }
2253 : }
2254 :
2255 : /*
2256 : * If the task is part of a workload that spans multiple NUMA nodes,
2257 : * and is migrating into one of the workload's active nodes, remember
2258 : * this node as the task's preferred numa node, so the workload can
2259 : * settle down.
2260 : * A task that migrated to a second choice node will be better off
2261 : * trying for a better one later. Do not set the preferred node here.
2262 : */
2263 : if (ng) {
2264 : if (env.best_cpu == -1)
2265 : nid = env.src_nid;
2266 : else
2267 : nid = cpu_to_node(env.best_cpu);
2268 :
2269 : if (nid != p->numa_preferred_nid)
2270 : sched_setnuma(p, nid);
2271 : }
2272 :
2273 : /* No better CPU than the current one was found. */
2274 : if (env.best_cpu == -1) {
2275 : trace_sched_stick_numa(p, env.src_cpu, NULL, -1);
2276 : return -EAGAIN;
2277 : }
2278 :
2279 : best_rq = cpu_rq(env.best_cpu);
2280 : if (env.best_task == NULL) {
2281 : ret = migrate_task_to(p, env.best_cpu);
2282 : WRITE_ONCE(best_rq->numa_migrate_on, 0);
2283 : if (ret != 0)
2284 : trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu);
2285 : return ret;
2286 : }
2287 :
2288 : ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu);
2289 : WRITE_ONCE(best_rq->numa_migrate_on, 0);
2290 :
2291 : if (ret != 0)
2292 : trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu);
2293 : put_task_struct(env.best_task);
2294 : return ret;
2295 : }
2296 :
2297 : /* Attempt to migrate a task to a CPU on the preferred node. */
2298 : static void numa_migrate_preferred(struct task_struct *p)
2299 : {
2300 : unsigned long interval = HZ;
2301 :
2302 : /* This task has no NUMA fault statistics yet */
2303 : if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults))
2304 : return;
2305 :
2306 : /* Periodically retry migrating the task to the preferred node */
2307 : interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
2308 : p->numa_migrate_retry = jiffies + interval;
2309 :
2310 : /* Success if task is already running on preferred CPU */
2311 : if (task_node(p) == p->numa_preferred_nid)
2312 : return;
2313 :
2314 : /* Otherwise, try migrate to a CPU on the preferred node */
2315 : task_numa_migrate(p);
2316 : }
2317 :
2318 : /*
2319 : * Find out how many nodes the workload is actively running on. Do this by
2320 : * tracking the nodes from which NUMA hinting faults are triggered. This can
2321 : * be different from the set of nodes where the workload's memory is currently
2322 : * located.
2323 : */
2324 : static void numa_group_count_active_nodes(struct numa_group *numa_group)
2325 : {
2326 : unsigned long faults, max_faults = 0;
2327 : int nid, active_nodes = 0;
2328 :
2329 : for_each_node_state(nid, N_CPU) {
2330 : faults = group_faults_cpu(numa_group, nid);
2331 : if (faults > max_faults)
2332 : max_faults = faults;
2333 : }
2334 :
2335 : for_each_node_state(nid, N_CPU) {
2336 : faults = group_faults_cpu(numa_group, nid);
2337 : if (faults * ACTIVE_NODE_FRACTION > max_faults)
2338 : active_nodes++;
2339 : }
2340 :
2341 : numa_group->max_faults_cpu = max_faults;
2342 : numa_group->active_nodes = active_nodes;
2343 : }
2344 :
2345 : /*
2346 : * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
2347 : * increments. The more local the fault statistics are, the higher the scan
2348 : * period will be for the next scan window. If local/(local+remote) ratio is
2349 : * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS)
2350 : * the scan period will decrease. Aim for 70% local accesses.
2351 : */
2352 : #define NUMA_PERIOD_SLOTS 10
2353 : #define NUMA_PERIOD_THRESHOLD 7
2354 :
2355 : /*
2356 : * Increase the scan period (slow down scanning) if the majority of
2357 : * our memory is already on our local node, or if the majority of
2358 : * the page accesses are shared with other processes.
2359 : * Otherwise, decrease the scan period.
2360 : */
2361 : static void update_task_scan_period(struct task_struct *p,
2362 : unsigned long shared, unsigned long private)
2363 : {
2364 : unsigned int period_slot;
2365 : int lr_ratio, ps_ratio;
2366 : int diff;
2367 :
2368 : unsigned long remote = p->numa_faults_locality[0];
2369 : unsigned long local = p->numa_faults_locality[1];
2370 :
2371 : /*
2372 : * If there were no record hinting faults then either the task is
2373 : * completely idle or all activity is in areas that are not of interest
2374 : * to automatic numa balancing. Related to that, if there were failed
2375 : * migration then it implies we are migrating too quickly or the local
2376 : * node is overloaded. In either case, scan slower
2377 : */
2378 : if (local + shared == 0 || p->numa_faults_locality[2]) {
2379 : p->numa_scan_period = min(p->numa_scan_period_max,
2380 : p->numa_scan_period << 1);
2381 :
2382 : p->mm->numa_next_scan = jiffies +
2383 : msecs_to_jiffies(p->numa_scan_period);
2384 :
2385 : return;
2386 : }
2387 :
2388 : /*
2389 : * Prepare to scale scan period relative to the current period.
2390 : * == NUMA_PERIOD_THRESHOLD scan period stays the same
2391 : * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
2392 : * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
2393 : */
2394 : period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
2395 : lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
2396 : ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared);
2397 :
2398 : if (ps_ratio >= NUMA_PERIOD_THRESHOLD) {
2399 : /*
2400 : * Most memory accesses are local. There is no need to
2401 : * do fast NUMA scanning, since memory is already local.
2402 : */
2403 : int slot = ps_ratio - NUMA_PERIOD_THRESHOLD;
2404 : if (!slot)
2405 : slot = 1;
2406 : diff = slot * period_slot;
2407 : } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) {
2408 : /*
2409 : * Most memory accesses are shared with other tasks.
2410 : * There is no point in continuing fast NUMA scanning,
2411 : * since other tasks may just move the memory elsewhere.
2412 : */
2413 : int slot = lr_ratio - NUMA_PERIOD_THRESHOLD;
2414 : if (!slot)
2415 : slot = 1;
2416 : diff = slot * period_slot;
2417 : } else {
2418 : /*
2419 : * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS,
2420 : * yet they are not on the local NUMA node. Speed up
2421 : * NUMA scanning to get the memory moved over.
2422 : */
2423 : int ratio = max(lr_ratio, ps_ratio);
2424 : diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
2425 : }
2426 :
2427 : p->numa_scan_period = clamp(p->numa_scan_period + diff,
2428 : task_scan_min(p), task_scan_max(p));
2429 : memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2430 : }
2431 :
2432 : /*
2433 : * Get the fraction of time the task has been running since the last
2434 : * NUMA placement cycle. The scheduler keeps similar statistics, but
2435 : * decays those on a 32ms period, which is orders of magnitude off
2436 : * from the dozens-of-seconds NUMA balancing period. Use the scheduler
2437 : * stats only if the task is so new there are no NUMA statistics yet.
2438 : */
2439 : static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
2440 : {
2441 : u64 runtime, delta, now;
2442 : /* Use the start of this time slice to avoid calculations. */
2443 : now = p->se.exec_start;
2444 : runtime = p->se.sum_exec_runtime;
2445 :
2446 : if (p->last_task_numa_placement) {
2447 : delta = runtime - p->last_sum_exec_runtime;
2448 : *period = now - p->last_task_numa_placement;
2449 :
2450 : /* Avoid time going backwards, prevent potential divide error: */
2451 : if (unlikely((s64)*period < 0))
2452 : *period = 0;
2453 : } else {
2454 : delta = p->se.avg.load_sum;
2455 : *period = LOAD_AVG_MAX;
2456 : }
2457 :
2458 : p->last_sum_exec_runtime = runtime;
2459 : p->last_task_numa_placement = now;
2460 :
2461 : return delta;
2462 : }
2463 :
2464 : /*
2465 : * Determine the preferred nid for a task in a numa_group. This needs to
2466 : * be done in a way that produces consistent results with group_weight,
2467 : * otherwise workloads might not converge.
2468 : */
2469 : static int preferred_group_nid(struct task_struct *p, int nid)
2470 : {
2471 : nodemask_t nodes;
2472 : int dist;
2473 :
2474 : /* Direct connections between all NUMA nodes. */
2475 : if (sched_numa_topology_type == NUMA_DIRECT)
2476 : return nid;
2477 :
2478 : /*
2479 : * On a system with glueless mesh NUMA topology, group_weight
2480 : * scores nodes according to the number of NUMA hinting faults on
2481 : * both the node itself, and on nearby nodes.
2482 : */
2483 : if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
2484 : unsigned long score, max_score = 0;
2485 : int node, max_node = nid;
2486 :
2487 : dist = sched_max_numa_distance;
2488 :
2489 : for_each_node_state(node, N_CPU) {
2490 : score = group_weight(p, node, dist);
2491 : if (score > max_score) {
2492 : max_score = score;
2493 : max_node = node;
2494 : }
2495 : }
2496 : return max_node;
2497 : }
2498 :
2499 : /*
2500 : * Finding the preferred nid in a system with NUMA backplane
2501 : * interconnect topology is more involved. The goal is to locate
2502 : * tasks from numa_groups near each other in the system, and
2503 : * untangle workloads from different sides of the system. This requires
2504 : * searching down the hierarchy of node groups, recursively searching
2505 : * inside the highest scoring group of nodes. The nodemask tricks
2506 : * keep the complexity of the search down.
2507 : */
2508 : nodes = node_states[N_CPU];
2509 : for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
2510 : unsigned long max_faults = 0;
2511 : nodemask_t max_group = NODE_MASK_NONE;
2512 : int a, b;
2513 :
2514 : /* Are there nodes at this distance from each other? */
2515 : if (!find_numa_distance(dist))
2516 : continue;
2517 :
2518 : for_each_node_mask(a, nodes) {
2519 : unsigned long faults = 0;
2520 : nodemask_t this_group;
2521 : nodes_clear(this_group);
2522 :
2523 : /* Sum group's NUMA faults; includes a==b case. */
2524 : for_each_node_mask(b, nodes) {
2525 : if (node_distance(a, b) < dist) {
2526 : faults += group_faults(p, b);
2527 : node_set(b, this_group);
2528 : node_clear(b, nodes);
2529 : }
2530 : }
2531 :
2532 : /* Remember the top group. */
2533 : if (faults > max_faults) {
2534 : max_faults = faults;
2535 : max_group = this_group;
2536 : /*
2537 : * subtle: at the smallest distance there is
2538 : * just one node left in each "group", the
2539 : * winner is the preferred nid.
2540 : */
2541 : nid = a;
2542 : }
2543 : }
2544 : /* Next round, evaluate the nodes within max_group. */
2545 : if (!max_faults)
2546 : break;
2547 : nodes = max_group;
2548 : }
2549 : return nid;
2550 : }
2551 :
2552 : static void task_numa_placement(struct task_struct *p)
2553 : {
2554 : int seq, nid, max_nid = NUMA_NO_NODE;
2555 : unsigned long max_faults = 0;
2556 : unsigned long fault_types[2] = { 0, 0 };
2557 : unsigned long total_faults;
2558 : u64 runtime, period;
2559 : spinlock_t *group_lock = NULL;
2560 : struct numa_group *ng;
2561 :
2562 : /*
2563 : * The p->mm->numa_scan_seq field gets updated without
2564 : * exclusive access. Use READ_ONCE() here to ensure
2565 : * that the field is read in a single access:
2566 : */
2567 : seq = READ_ONCE(p->mm->numa_scan_seq);
2568 : if (p->numa_scan_seq == seq)
2569 : return;
2570 : p->numa_scan_seq = seq;
2571 : p->numa_scan_period_max = task_scan_max(p);
2572 :
2573 : total_faults = p->numa_faults_locality[0] +
2574 : p->numa_faults_locality[1];
2575 : runtime = numa_get_avg_runtime(p, &period);
2576 :
2577 : /* If the task is part of a group prevent parallel updates to group stats */
2578 : ng = deref_curr_numa_group(p);
2579 : if (ng) {
2580 : group_lock = &ng->lock;
2581 : spin_lock_irq(group_lock);
2582 : }
2583 :
2584 : /* Find the node with the highest number of faults */
2585 : for_each_online_node(nid) {
2586 : /* Keep track of the offsets in numa_faults array */
2587 : int mem_idx, membuf_idx, cpu_idx, cpubuf_idx;
2588 : unsigned long faults = 0, group_faults = 0;
2589 : int priv;
2590 :
2591 : for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
2592 : long diff, f_diff, f_weight;
2593 :
2594 : mem_idx = task_faults_idx(NUMA_MEM, nid, priv);
2595 : membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);
2596 : cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);
2597 : cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);
2598 :
2599 : /* Decay existing window, copy faults since last scan */
2600 : diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2;
2601 : fault_types[priv] += p->numa_faults[membuf_idx];
2602 : p->numa_faults[membuf_idx] = 0;
2603 :
2604 : /*
2605 : * Normalize the faults_from, so all tasks in a group
2606 : * count according to CPU use, instead of by the raw
2607 : * number of faults. Tasks with little runtime have
2608 : * little over-all impact on throughput, and thus their
2609 : * faults are less important.
2610 : */
2611 : f_weight = div64_u64(runtime << 16, period + 1);
2612 : f_weight = (f_weight * p->numa_faults[cpubuf_idx]) /
2613 : (total_faults + 1);
2614 : f_diff = f_weight - p->numa_faults[cpu_idx] / 2;
2615 : p->numa_faults[cpubuf_idx] = 0;
2616 :
2617 : p->numa_faults[mem_idx] += diff;
2618 : p->numa_faults[cpu_idx] += f_diff;
2619 : faults += p->numa_faults[mem_idx];
2620 : p->total_numa_faults += diff;
2621 : if (ng) {
2622 : /*
2623 : * safe because we can only change our own group
2624 : *
2625 : * mem_idx represents the offset for a given
2626 : * nid and priv in a specific region because it
2627 : * is at the beginning of the numa_faults array.
2628 : */
2629 : ng->faults[mem_idx] += diff;
2630 : ng->faults[cpu_idx] += f_diff;
2631 : ng->total_faults += diff;
2632 : group_faults += ng->faults[mem_idx];
2633 : }
2634 : }
2635 :
2636 : if (!ng) {
2637 : if (faults > max_faults) {
2638 : max_faults = faults;
2639 : max_nid = nid;
2640 : }
2641 : } else if (group_faults > max_faults) {
2642 : max_faults = group_faults;
2643 : max_nid = nid;
2644 : }
2645 : }
2646 :
2647 : /* Cannot migrate task to CPU-less node */
2648 : if (max_nid != NUMA_NO_NODE && !node_state(max_nid, N_CPU)) {
2649 : int near_nid = max_nid;
2650 : int distance, near_distance = INT_MAX;
2651 :
2652 : for_each_node_state(nid, N_CPU) {
2653 : distance = node_distance(max_nid, nid);
2654 : if (distance < near_distance) {
2655 : near_nid = nid;
2656 : near_distance = distance;
2657 : }
2658 : }
2659 : max_nid = near_nid;
2660 : }
2661 :
2662 : if (ng) {
2663 : numa_group_count_active_nodes(ng);
2664 : spin_unlock_irq(group_lock);
2665 : max_nid = preferred_group_nid(p, max_nid);
2666 : }
2667 :
2668 : if (max_faults) {
2669 : /* Set the new preferred node */
2670 : if (max_nid != p->numa_preferred_nid)
2671 : sched_setnuma(p, max_nid);
2672 : }
2673 :
2674 : update_task_scan_period(p, fault_types[0], fault_types[1]);
2675 : }
2676 :
2677 : static inline int get_numa_group(struct numa_group *grp)
2678 : {
2679 : return refcount_inc_not_zero(&grp->refcount);
2680 : }
2681 :
2682 : static inline void put_numa_group(struct numa_group *grp)
2683 : {
2684 : if (refcount_dec_and_test(&grp->refcount))
2685 : kfree_rcu(grp, rcu);
2686 : }
2687 :
2688 : static void task_numa_group(struct task_struct *p, int cpupid, int flags,
2689 : int *priv)
2690 : {
2691 : struct numa_group *grp, *my_grp;
2692 : struct task_struct *tsk;
2693 : bool join = false;
2694 : int cpu = cpupid_to_cpu(cpupid);
2695 : int i;
2696 :
2697 : if (unlikely(!deref_curr_numa_group(p))) {
2698 : unsigned int size = sizeof(struct numa_group) +
2699 : NR_NUMA_HINT_FAULT_STATS *
2700 : nr_node_ids * sizeof(unsigned long);
2701 :
2702 : grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
2703 : if (!grp)
2704 : return;
2705 :
2706 : refcount_set(&grp->refcount, 1);
2707 : grp->active_nodes = 1;
2708 : grp->max_faults_cpu = 0;
2709 : spin_lock_init(&grp->lock);
2710 : grp->gid = p->pid;
2711 :
2712 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2713 : grp->faults[i] = p->numa_faults[i];
2714 :
2715 : grp->total_faults = p->total_numa_faults;
2716 :
2717 : grp->nr_tasks++;
2718 : rcu_assign_pointer(p->numa_group, grp);
2719 : }
2720 :
2721 : rcu_read_lock();
2722 : tsk = READ_ONCE(cpu_rq(cpu)->curr);
2723 :
2724 : if (!cpupid_match_pid(tsk, cpupid))
2725 : goto no_join;
2726 :
2727 : grp = rcu_dereference(tsk->numa_group);
2728 : if (!grp)
2729 : goto no_join;
2730 :
2731 : my_grp = deref_curr_numa_group(p);
2732 : if (grp == my_grp)
2733 : goto no_join;
2734 :
2735 : /*
2736 : * Only join the other group if its bigger; if we're the bigger group,
2737 : * the other task will join us.
2738 : */
2739 : if (my_grp->nr_tasks > grp->nr_tasks)
2740 : goto no_join;
2741 :
2742 : /*
2743 : * Tie-break on the grp address.
2744 : */
2745 : if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
2746 : goto no_join;
2747 :
2748 : /* Always join threads in the same process. */
2749 : if (tsk->mm == current->mm)
2750 : join = true;
2751 :
2752 : /* Simple filter to avoid false positives due to PID collisions */
2753 : if (flags & TNF_SHARED)
2754 : join = true;
2755 :
2756 : /* Update priv based on whether false sharing was detected */
2757 : *priv = !join;
2758 :
2759 : if (join && !get_numa_group(grp))
2760 : goto no_join;
2761 :
2762 : rcu_read_unlock();
2763 :
2764 : if (!join)
2765 : return;
2766 :
2767 : WARN_ON_ONCE(irqs_disabled());
2768 : double_lock_irq(&my_grp->lock, &grp->lock);
2769 :
2770 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
2771 : my_grp->faults[i] -= p->numa_faults[i];
2772 : grp->faults[i] += p->numa_faults[i];
2773 : }
2774 : my_grp->total_faults -= p->total_numa_faults;
2775 : grp->total_faults += p->total_numa_faults;
2776 :
2777 : my_grp->nr_tasks--;
2778 : grp->nr_tasks++;
2779 :
2780 : spin_unlock(&my_grp->lock);
2781 : spin_unlock_irq(&grp->lock);
2782 :
2783 : rcu_assign_pointer(p->numa_group, grp);
2784 :
2785 : put_numa_group(my_grp);
2786 : return;
2787 :
2788 : no_join:
2789 : rcu_read_unlock();
2790 : return;
2791 : }
2792 :
2793 : /*
2794 : * Get rid of NUMA statistics associated with a task (either current or dead).
2795 : * If @final is set, the task is dead and has reached refcount zero, so we can
2796 : * safely free all relevant data structures. Otherwise, there might be
2797 : * concurrent reads from places like load balancing and procfs, and we should
2798 : * reset the data back to default state without freeing ->numa_faults.
2799 : */
2800 : void task_numa_free(struct task_struct *p, bool final)
2801 : {
2802 : /* safe: p either is current or is being freed by current */
2803 : struct numa_group *grp = rcu_dereference_raw(p->numa_group);
2804 : unsigned long *numa_faults = p->numa_faults;
2805 : unsigned long flags;
2806 : int i;
2807 :
2808 : if (!numa_faults)
2809 : return;
2810 :
2811 : if (grp) {
2812 : spin_lock_irqsave(&grp->lock, flags);
2813 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2814 : grp->faults[i] -= p->numa_faults[i];
2815 : grp->total_faults -= p->total_numa_faults;
2816 :
2817 : grp->nr_tasks--;
2818 : spin_unlock_irqrestore(&grp->lock, flags);
2819 : RCU_INIT_POINTER(p->numa_group, NULL);
2820 : put_numa_group(grp);
2821 : }
2822 :
2823 : if (final) {
2824 : p->numa_faults = NULL;
2825 : kfree(numa_faults);
2826 : } else {
2827 : p->total_numa_faults = 0;
2828 : for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2829 : numa_faults[i] = 0;
2830 : }
2831 : }
2832 :
2833 : /*
2834 : * Got a PROT_NONE fault for a page on @node.
2835 : */
2836 : void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
2837 : {
2838 : struct task_struct *p = current;
2839 : bool migrated = flags & TNF_MIGRATED;
2840 : int cpu_node = task_node(current);
2841 : int local = !!(flags & TNF_FAULT_LOCAL);
2842 : struct numa_group *ng;
2843 : int priv;
2844 :
2845 : if (!static_branch_likely(&sched_numa_balancing))
2846 : return;
2847 :
2848 : /* for example, ksmd faulting in a user's mm */
2849 : if (!p->mm)
2850 : return;
2851 :
2852 : /*
2853 : * NUMA faults statistics are unnecessary for the slow memory
2854 : * node for memory tiering mode.
2855 : */
2856 : if (!node_is_toptier(mem_node) &&
2857 : (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING ||
2858 : !cpupid_valid(last_cpupid)))
2859 : return;
2860 :
2861 : /* Allocate buffer to track faults on a per-node basis */
2862 : if (unlikely(!p->numa_faults)) {
2863 : int size = sizeof(*p->numa_faults) *
2864 : NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
2865 :
2866 : p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN);
2867 : if (!p->numa_faults)
2868 : return;
2869 :
2870 : p->total_numa_faults = 0;
2871 : memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2872 : }
2873 :
2874 : /*
2875 : * First accesses are treated as private, otherwise consider accesses
2876 : * to be private if the accessing pid has not changed
2877 : */
2878 : if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
2879 : priv = 1;
2880 : } else {
2881 : priv = cpupid_match_pid(p, last_cpupid);
2882 : if (!priv && !(flags & TNF_NO_GROUP))
2883 : task_numa_group(p, last_cpupid, flags, &priv);
2884 : }
2885 :
2886 : /*
2887 : * If a workload spans multiple NUMA nodes, a shared fault that
2888 : * occurs wholly within the set of nodes that the workload is
2889 : * actively using should be counted as local. This allows the
2890 : * scan rate to slow down when a workload has settled down.
2891 : */
2892 : ng = deref_curr_numa_group(p);
2893 : if (!priv && !local && ng && ng->active_nodes > 1 &&
2894 : numa_is_active_node(cpu_node, ng) &&
2895 : numa_is_active_node(mem_node, ng))
2896 : local = 1;
2897 :
2898 : /*
2899 : * Retry to migrate task to preferred node periodically, in case it
2900 : * previously failed, or the scheduler moved us.
2901 : */
2902 : if (time_after(jiffies, p->numa_migrate_retry)) {
2903 : task_numa_placement(p);
2904 : numa_migrate_preferred(p);
2905 : }
2906 :
2907 : if (migrated)
2908 : p->numa_pages_migrated += pages;
2909 : if (flags & TNF_MIGRATE_FAIL)
2910 : p->numa_faults_locality[2] += pages;
2911 :
2912 : p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages;
2913 : p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages;
2914 : p->numa_faults_locality[local] += pages;
2915 : }
2916 :
2917 : static void reset_ptenuma_scan(struct task_struct *p)
2918 : {
2919 : /*
2920 : * We only did a read acquisition of the mmap sem, so
2921 : * p->mm->numa_scan_seq is written to without exclusive access
2922 : * and the update is not guaranteed to be atomic. That's not
2923 : * much of an issue though, since this is just used for
2924 : * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
2925 : * expensive, to avoid any form of compiler optimizations:
2926 : */
2927 : WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
2928 : p->mm->numa_scan_offset = 0;
2929 : }
2930 :
2931 : /*
2932 : * The expensive part of numa migration is done from task_work context.
2933 : * Triggered from task_tick_numa().
2934 : */
2935 : static void task_numa_work(struct callback_head *work)
2936 : {
2937 : unsigned long migrate, next_scan, now = jiffies;
2938 : struct task_struct *p = current;
2939 : struct mm_struct *mm = p->mm;
2940 : u64 runtime = p->se.sum_exec_runtime;
2941 : struct vm_area_struct *vma;
2942 : unsigned long start, end;
2943 : unsigned long nr_pte_updates = 0;
2944 : long pages, virtpages;
2945 : struct vma_iterator vmi;
2946 :
2947 : SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work));
2948 :
2949 : work->next = work;
2950 : /*
2951 : * Who cares about NUMA placement when they're dying.
2952 : *
2953 : * NOTE: make sure not to dereference p->mm before this check,
2954 : * exit_task_work() happens _after_ exit_mm() so we could be called
2955 : * without p->mm even though we still had it when we enqueued this
2956 : * work.
2957 : */
2958 : if (p->flags & PF_EXITING)
2959 : return;
2960 :
2961 : if (!mm->numa_next_scan) {
2962 : mm->numa_next_scan = now +
2963 : msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2964 : }
2965 :
2966 : /*
2967 : * Enforce maximal scan/migration frequency..
2968 : */
2969 : migrate = mm->numa_next_scan;
2970 : if (time_before(now, migrate))
2971 : return;
2972 :
2973 : if (p->numa_scan_period == 0) {
2974 : p->numa_scan_period_max = task_scan_max(p);
2975 : p->numa_scan_period = task_scan_start(p);
2976 : }
2977 :
2978 : next_scan = now + msecs_to_jiffies(p->numa_scan_period);
2979 : if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan))
2980 : return;
2981 :
2982 : /*
2983 : * Delay this task enough that another task of this mm will likely win
2984 : * the next time around.
2985 : */
2986 : p->node_stamp += 2 * TICK_NSEC;
2987 :
2988 : start = mm->numa_scan_offset;
2989 : pages = sysctl_numa_balancing_scan_size;
2990 : pages <<= 20 - PAGE_SHIFT; /* MB in pages */
2991 : virtpages = pages * 8; /* Scan up to this much virtual space */
2992 : if (!pages)
2993 : return;
2994 :
2995 :
2996 : if (!mmap_read_trylock(mm))
2997 : return;
2998 : vma_iter_init(&vmi, mm, start);
2999 : vma = vma_next(&vmi);
3000 : if (!vma) {
3001 : reset_ptenuma_scan(p);
3002 : start = 0;
3003 : vma_iter_set(&vmi, start);
3004 : vma = vma_next(&vmi);
3005 : }
3006 :
3007 : do {
3008 : if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
3009 : is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
3010 : continue;
3011 : }
3012 :
3013 : /*
3014 : * Shared library pages mapped by multiple processes are not
3015 : * migrated as it is expected they are cache replicated. Avoid
3016 : * hinting faults in read-only file-backed mappings or the vdso
3017 : * as migrating the pages will be of marginal benefit.
3018 : */
3019 : if (!vma->vm_mm ||
3020 : (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
3021 : continue;
3022 :
3023 : /*
3024 : * Skip inaccessible VMAs to avoid any confusion between
3025 : * PROT_NONE and NUMA hinting ptes
3026 : */
3027 : if (!vma_is_accessible(vma))
3028 : continue;
3029 :
3030 : do {
3031 : start = max(start, vma->vm_start);
3032 : end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
3033 : end = min(end, vma->vm_end);
3034 : nr_pte_updates = change_prot_numa(vma, start, end);
3035 :
3036 : /*
3037 : * Try to scan sysctl_numa_balancing_size worth of
3038 : * hpages that have at least one present PTE that
3039 : * is not already pte-numa. If the VMA contains
3040 : * areas that are unused or already full of prot_numa
3041 : * PTEs, scan up to virtpages, to skip through those
3042 : * areas faster.
3043 : */
3044 : if (nr_pte_updates)
3045 : pages -= (end - start) >> PAGE_SHIFT;
3046 : virtpages -= (end - start) >> PAGE_SHIFT;
3047 :
3048 : start = end;
3049 : if (pages <= 0 || virtpages <= 0)
3050 : goto out;
3051 :
3052 : cond_resched();
3053 : } while (end != vma->vm_end);
3054 : } for_each_vma(vmi, vma);
3055 :
3056 : out:
3057 : /*
3058 : * It is possible to reach the end of the VMA list but the last few
3059 : * VMAs are not guaranteed to the vma_migratable. If they are not, we
3060 : * would find the !migratable VMA on the next scan but not reset the
3061 : * scanner to the start so check it now.
3062 : */
3063 : if (vma)
3064 : mm->numa_scan_offset = start;
3065 : else
3066 : reset_ptenuma_scan(p);
3067 : mmap_read_unlock(mm);
3068 :
3069 : /*
3070 : * Make sure tasks use at least 32x as much time to run other code
3071 : * than they used here, to limit NUMA PTE scanning overhead to 3% max.
3072 : * Usually update_task_scan_period slows down scanning enough; on an
3073 : * overloaded system we need to limit overhead on a per task basis.
3074 : */
3075 : if (unlikely(p->se.sum_exec_runtime != runtime)) {
3076 : u64 diff = p->se.sum_exec_runtime - runtime;
3077 : p->node_stamp += 32 * diff;
3078 : }
3079 : }
3080 :
3081 : void init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
3082 : {
3083 : int mm_users = 0;
3084 : struct mm_struct *mm = p->mm;
3085 :
3086 : if (mm) {
3087 : mm_users = atomic_read(&mm->mm_users);
3088 : if (mm_users == 1) {
3089 : mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
3090 : mm->numa_scan_seq = 0;
3091 : }
3092 : }
3093 : p->node_stamp = 0;
3094 : p->numa_scan_seq = mm ? mm->numa_scan_seq : 0;
3095 : p->numa_scan_period = sysctl_numa_balancing_scan_delay;
3096 : p->numa_migrate_retry = 0;
3097 : /* Protect against double add, see task_tick_numa and task_numa_work */
3098 : p->numa_work.next = &p->numa_work;
3099 : p->numa_faults = NULL;
3100 : p->numa_pages_migrated = 0;
3101 : p->total_numa_faults = 0;
3102 : RCU_INIT_POINTER(p->numa_group, NULL);
3103 : p->last_task_numa_placement = 0;
3104 : p->last_sum_exec_runtime = 0;
3105 :
3106 : init_task_work(&p->numa_work, task_numa_work);
3107 :
3108 : /* New address space, reset the preferred nid */
3109 : if (!(clone_flags & CLONE_VM)) {
3110 : p->numa_preferred_nid = NUMA_NO_NODE;
3111 : return;
3112 : }
3113 :
3114 : /*
3115 : * New thread, keep existing numa_preferred_nid which should be copied
3116 : * already by arch_dup_task_struct but stagger when scans start.
3117 : */
3118 : if (mm) {
3119 : unsigned int delay;
3120 :
3121 : delay = min_t(unsigned int, task_scan_max(current),
3122 : current->numa_scan_period * mm_users * NSEC_PER_MSEC);
3123 : delay += 2 * TICK_NSEC;
3124 : p->node_stamp = delay;
3125 : }
3126 : }
3127 :
3128 : /*
3129 : * Drive the periodic memory faults..
3130 : */
3131 : static void task_tick_numa(struct rq *rq, struct task_struct *curr)
3132 : {
3133 : struct callback_head *work = &curr->numa_work;
3134 : u64 period, now;
3135 :
3136 : /*
3137 : * We don't care about NUMA placement if we don't have memory.
3138 : */
3139 : if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work)
3140 : return;
3141 :
3142 : /*
3143 : * Using runtime rather than walltime has the dual advantage that
3144 : * we (mostly) drive the selection from busy threads and that the
3145 : * task needs to have done some actual work before we bother with
3146 : * NUMA placement.
3147 : */
3148 : now = curr->se.sum_exec_runtime;
3149 : period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
3150 :
3151 : if (now > curr->node_stamp + period) {
3152 : if (!curr->node_stamp)
3153 : curr->numa_scan_period = task_scan_start(curr);
3154 : curr->node_stamp += period;
3155 :
3156 : if (!time_before(jiffies, curr->mm->numa_next_scan))
3157 : task_work_add(curr, work, TWA_RESUME);
3158 : }
3159 : }
3160 :
3161 : static void update_scan_period(struct task_struct *p, int new_cpu)
3162 : {
3163 : int src_nid = cpu_to_node(task_cpu(p));
3164 : int dst_nid = cpu_to_node(new_cpu);
3165 :
3166 : if (!static_branch_likely(&sched_numa_balancing))
3167 : return;
3168 :
3169 : if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING))
3170 : return;
3171 :
3172 : if (src_nid == dst_nid)
3173 : return;
3174 :
3175 : /*
3176 : * Allow resets if faults have been trapped before one scan
3177 : * has completed. This is most likely due to a new task that
3178 : * is pulled cross-node due to wakeups or load balancing.
3179 : */
3180 : if (p->numa_scan_seq) {
3181 : /*
3182 : * Avoid scan adjustments if moving to the preferred
3183 : * node or if the task was not previously running on
3184 : * the preferred node.
3185 : */
3186 : if (dst_nid == p->numa_preferred_nid ||
3187 : (p->numa_preferred_nid != NUMA_NO_NODE &&
3188 : src_nid != p->numa_preferred_nid))
3189 : return;
3190 : }
3191 :
3192 : p->numa_scan_period = task_scan_start(p);
3193 : }
3194 :
3195 : #else
3196 : static void task_tick_numa(struct rq *rq, struct task_struct *curr)
3197 : {
3198 : }
3199 :
3200 : static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
3201 : {
3202 : }
3203 :
3204 : static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
3205 : {
3206 : }
3207 :
3208 : static inline void update_scan_period(struct task_struct *p, int new_cpu)
3209 : {
3210 : }
3211 :
3212 : #endif /* CONFIG_NUMA_BALANCING */
3213 :
3214 : static void
3215 : account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3216 : {
3217 4264 : update_load_add(&cfs_rq->load, se->load.weight);
3218 : #ifdef CONFIG_SMP
3219 : if (entity_is_task(se)) {
3220 : struct rq *rq = rq_of(cfs_rq);
3221 :
3222 : account_numa_enqueue(rq, task_of(se));
3223 : list_add(&se->group_node, &rq->cfs_tasks);
3224 : }
3225 : #endif
3226 2132 : cfs_rq->nr_running++;
3227 2132 : if (se_is_idle(se))
3228 : cfs_rq->idle_nr_running++;
3229 : }
3230 :
3231 : static void
3232 : account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3233 : {
3234 4260 : update_load_sub(&cfs_rq->load, se->load.weight);
3235 : #ifdef CONFIG_SMP
3236 : if (entity_is_task(se)) {
3237 : account_numa_dequeue(rq_of(cfs_rq), task_of(se));
3238 : list_del_init(&se->group_node);
3239 : }
3240 : #endif
3241 2130 : cfs_rq->nr_running--;
3242 2130 : if (se_is_idle(se))
3243 : cfs_rq->idle_nr_running--;
3244 : }
3245 :
3246 : /*
3247 : * Signed add and clamp on underflow.
3248 : *
3249 : * Explicitly do a load-store to ensure the intermediate value never hits
3250 : * memory. This allows lockless observations without ever seeing the negative
3251 : * values.
3252 : */
3253 : #define add_positive(_ptr, _val) do { \
3254 : typeof(_ptr) ptr = (_ptr); \
3255 : typeof(_val) val = (_val); \
3256 : typeof(*ptr) res, var = READ_ONCE(*ptr); \
3257 : \
3258 : res = var + val; \
3259 : \
3260 : if (val < 0 && res > var) \
3261 : res = 0; \
3262 : \
3263 : WRITE_ONCE(*ptr, res); \
3264 : } while (0)
3265 :
3266 : /*
3267 : * Unsigned subtract and clamp on underflow.
3268 : *
3269 : * Explicitly do a load-store to ensure the intermediate value never hits
3270 : * memory. This allows lockless observations without ever seeing the negative
3271 : * values.
3272 : */
3273 : #define sub_positive(_ptr, _val) do { \
3274 : typeof(_ptr) ptr = (_ptr); \
3275 : typeof(*ptr) val = (_val); \
3276 : typeof(*ptr) res, var = READ_ONCE(*ptr); \
3277 : res = var - val; \
3278 : if (res > var) \
3279 : res = 0; \
3280 : WRITE_ONCE(*ptr, res); \
3281 : } while (0)
3282 :
3283 : /*
3284 : * Remove and clamp on negative, from a local variable.
3285 : *
3286 : * A variant of sub_positive(), which does not use explicit load-store
3287 : * and is thus optimized for local variable updates.
3288 : */
3289 : #define lsub_positive(_ptr, _val) do { \
3290 : typeof(_ptr) ptr = (_ptr); \
3291 : *ptr -= min_t(typeof(*ptr), *ptr, _val); \
3292 : } while (0)
3293 :
3294 : #ifdef CONFIG_SMP
3295 : static inline void
3296 : enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3297 : {
3298 : cfs_rq->avg.load_avg += se->avg.load_avg;
3299 : cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;
3300 : }
3301 :
3302 : static inline void
3303 : dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3304 : {
3305 : sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
3306 : sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);
3307 : /* See update_cfs_rq_load_avg() */
3308 : cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
3309 : cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
3310 : }
3311 : #else
3312 : static inline void
3313 : enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3314 : static inline void
3315 : dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3316 : #endif
3317 :
3318 5 : static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
3319 : unsigned long weight)
3320 : {
3321 5 : if (se->on_rq) {
3322 : /* commit outstanding execution time */
3323 0 : if (cfs_rq->curr == se)
3324 0 : update_curr(cfs_rq);
3325 0 : update_load_sub(&cfs_rq->load, se->load.weight);
3326 : }
3327 5 : dequeue_load_avg(cfs_rq, se);
3328 :
3329 10 : update_load_set(&se->load, weight);
3330 :
3331 : #ifdef CONFIG_SMP
3332 : do {
3333 : u32 divider = get_pelt_divider(&se->avg);
3334 :
3335 : se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
3336 : } while (0);
3337 : #endif
3338 :
3339 5 : enqueue_load_avg(cfs_rq, se);
3340 5 : if (se->on_rq)
3341 0 : update_load_add(&cfs_rq->load, se->load.weight);
3342 :
3343 5 : }
3344 :
3345 5 : void reweight_task(struct task_struct *p, int prio)
3346 : {
3347 5 : struct sched_entity *se = &p->se;
3348 10 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
3349 5 : struct load_weight *load = &se->load;
3350 5 : unsigned long weight = scale_load(sched_prio_to_weight[prio]);
3351 :
3352 5 : reweight_entity(cfs_rq, se, weight);
3353 5 : load->inv_weight = sched_prio_to_wmult[prio];
3354 5 : }
3355 :
3356 : static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
3357 :
3358 : #ifdef CONFIG_FAIR_GROUP_SCHED
3359 : #ifdef CONFIG_SMP
3360 : /*
3361 : * All this does is approximate the hierarchical proportion which includes that
3362 : * global sum we all love to hate.
3363 : *
3364 : * That is, the weight of a group entity, is the proportional share of the
3365 : * group weight based on the group runqueue weights. That is:
3366 : *
3367 : * tg->weight * grq->load.weight
3368 : * ge->load.weight = ----------------------------- (1)
3369 : * \Sum grq->load.weight
3370 : *
3371 : * Now, because computing that sum is prohibitively expensive to compute (been
3372 : * there, done that) we approximate it with this average stuff. The average
3373 : * moves slower and therefore the approximation is cheaper and more stable.
3374 : *
3375 : * So instead of the above, we substitute:
3376 : *
3377 : * grq->load.weight -> grq->avg.load_avg (2)
3378 : *
3379 : * which yields the following:
3380 : *
3381 : * tg->weight * grq->avg.load_avg
3382 : * ge->load.weight = ------------------------------ (3)
3383 : * tg->load_avg
3384 : *
3385 : * Where: tg->load_avg ~= \Sum grq->avg.load_avg
3386 : *
3387 : * That is shares_avg, and it is right (given the approximation (2)).
3388 : *
3389 : * The problem with it is that because the average is slow -- it was designed
3390 : * to be exactly that of course -- this leads to transients in boundary
3391 : * conditions. In specific, the case where the group was idle and we start the
3392 : * one task. It takes time for our CPU's grq->avg.load_avg to build up,
3393 : * yielding bad latency etc..
3394 : *
3395 : * Now, in that special case (1) reduces to:
3396 : *
3397 : * tg->weight * grq->load.weight
3398 : * ge->load.weight = ----------------------------- = tg->weight (4)
3399 : * grp->load.weight
3400 : *
3401 : * That is, the sum collapses because all other CPUs are idle; the UP scenario.
3402 : *
3403 : * So what we do is modify our approximation (3) to approach (4) in the (near)
3404 : * UP case, like:
3405 : *
3406 : * ge->load.weight =
3407 : *
3408 : * tg->weight * grq->load.weight
3409 : * --------------------------------------------------- (5)
3410 : * tg->load_avg - grq->avg.load_avg + grq->load.weight
3411 : *
3412 : * But because grq->load.weight can drop to 0, resulting in a divide by zero,
3413 : * we need to use grq->avg.load_avg as its lower bound, which then gives:
3414 : *
3415 : *
3416 : * tg->weight * grq->load.weight
3417 : * ge->load.weight = ----------------------------- (6)
3418 : * tg_load_avg'
3419 : *
3420 : * Where:
3421 : *
3422 : * tg_load_avg' = tg->load_avg - grq->avg.load_avg +
3423 : * max(grq->load.weight, grq->avg.load_avg)
3424 : *
3425 : * And that is shares_weight and is icky. In the (near) UP case it approaches
3426 : * (4) while in the normal case it approaches (3). It consistently
3427 : * overestimates the ge->load.weight and therefore:
3428 : *
3429 : * \Sum ge->load.weight >= tg->weight
3430 : *
3431 : * hence icky!
3432 : */
3433 : static long calc_group_shares(struct cfs_rq *cfs_rq)
3434 : {
3435 : long tg_weight, tg_shares, load, shares;
3436 : struct task_group *tg = cfs_rq->tg;
3437 :
3438 : tg_shares = READ_ONCE(tg->shares);
3439 :
3440 : load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);
3441 :
3442 : tg_weight = atomic_long_read(&tg->load_avg);
3443 :
3444 : /* Ensure tg_weight >= load */
3445 : tg_weight -= cfs_rq->tg_load_avg_contrib;
3446 : tg_weight += load;
3447 :
3448 : shares = (tg_shares * load);
3449 : if (tg_weight)
3450 : shares /= tg_weight;
3451 :
3452 : /*
3453 : * MIN_SHARES has to be unscaled here to support per-CPU partitioning
3454 : * of a group with small tg->shares value. It is a floor value which is
3455 : * assigned as a minimum load.weight to the sched_entity representing
3456 : * the group on a CPU.
3457 : *
3458 : * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024
3459 : * on an 8-core system with 8 tasks each runnable on one CPU shares has
3460 : * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In
3461 : * case no task is runnable on a CPU MIN_SHARES=2 should be returned
3462 : * instead of 0.
3463 : */
3464 : return clamp_t(long, shares, MIN_SHARES, tg_shares);
3465 : }
3466 : #endif /* CONFIG_SMP */
3467 :
3468 : /*
3469 : * Recomputes the group entity based on the current state of its group
3470 : * runqueue.
3471 : */
3472 : static void update_cfs_group(struct sched_entity *se)
3473 : {
3474 : struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3475 : long shares;
3476 :
3477 : if (!gcfs_rq)
3478 : return;
3479 :
3480 : if (throttled_hierarchy(gcfs_rq))
3481 : return;
3482 :
3483 : #ifndef CONFIG_SMP
3484 : shares = READ_ONCE(gcfs_rq->tg->shares);
3485 :
3486 : if (likely(se->load.weight == shares))
3487 : return;
3488 : #else
3489 : shares = calc_group_shares(gcfs_rq);
3490 : #endif
3491 :
3492 : reweight_entity(cfs_rq_of(se), se, shares);
3493 : }
3494 :
3495 : #else /* CONFIG_FAIR_GROUP_SCHED */
3496 : static inline void update_cfs_group(struct sched_entity *se)
3497 : {
3498 : }
3499 : #endif /* CONFIG_FAIR_GROUP_SCHED */
3500 :
3501 : static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags)
3502 : {
3503 6981 : struct rq *rq = rq_of(cfs_rq);
3504 :
3505 : if (&rq->cfs == cfs_rq) {
3506 : /*
3507 : * There are a few boundary cases this might miss but it should
3508 : * get called often enough that that should (hopefully) not be
3509 : * a real problem.
3510 : *
3511 : * It will not get called when we go idle, because the idle
3512 : * thread is a different class (!fair), nor will the utilization
3513 : * number include things like RT tasks.
3514 : *
3515 : * As is, the util number is not freq-invariant (we'd have to
3516 : * implement arch_scale_freq_capacity() for that).
3517 : *
3518 : * See cpu_util_cfs().
3519 : */
3520 : cpufreq_update_util(rq, flags);
3521 : }
3522 : }
3523 :
3524 : #ifdef CONFIG_SMP
3525 : static inline bool load_avg_is_decayed(struct sched_avg *sa)
3526 : {
3527 : if (sa->load_sum)
3528 : return false;
3529 :
3530 : if (sa->util_sum)
3531 : return false;
3532 :
3533 : if (sa->runnable_sum)
3534 : return false;
3535 :
3536 : /*
3537 : * _avg must be null when _sum are null because _avg = _sum / divider
3538 : * Make sure that rounding and/or propagation of PELT values never
3539 : * break this.
3540 : */
3541 : SCHED_WARN_ON(sa->load_avg ||
3542 : sa->util_avg ||
3543 : sa->runnable_avg);
3544 :
3545 : return true;
3546 : }
3547 :
3548 : static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
3549 : {
3550 : return u64_u32_load_copy(cfs_rq->avg.last_update_time,
3551 : cfs_rq->last_update_time_copy);
3552 : }
3553 : #ifdef CONFIG_FAIR_GROUP_SCHED
3554 : /*
3555 : * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list
3556 : * immediately before a parent cfs_rq, and cfs_rqs are removed from the list
3557 : * bottom-up, we only have to test whether the cfs_rq before us on the list
3558 : * is our child.
3559 : * If cfs_rq is not on the list, test whether a child needs its to be added to
3560 : * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details).
3561 : */
3562 : static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq)
3563 : {
3564 : struct cfs_rq *prev_cfs_rq;
3565 : struct list_head *prev;
3566 :
3567 : if (cfs_rq->on_list) {
3568 : prev = cfs_rq->leaf_cfs_rq_list.prev;
3569 : } else {
3570 : struct rq *rq = rq_of(cfs_rq);
3571 :
3572 : prev = rq->tmp_alone_branch;
3573 : }
3574 :
3575 : prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list);
3576 :
3577 : return (prev_cfs_rq->tg->parent == cfs_rq->tg);
3578 : }
3579 :
3580 : static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
3581 : {
3582 : if (cfs_rq->load.weight)
3583 : return false;
3584 :
3585 : if (!load_avg_is_decayed(&cfs_rq->avg))
3586 : return false;
3587 :
3588 : if (child_cfs_rq_on_list(cfs_rq))
3589 : return false;
3590 :
3591 : return true;
3592 : }
3593 :
3594 : /**
3595 : * update_tg_load_avg - update the tg's load avg
3596 : * @cfs_rq: the cfs_rq whose avg changed
3597 : *
3598 : * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load.
3599 : * However, because tg->load_avg is a global value there are performance
3600 : * considerations.
3601 : *
3602 : * In order to avoid having to look at the other cfs_rq's, we use a
3603 : * differential update where we store the last value we propagated. This in
3604 : * turn allows skipping updates if the differential is 'small'.
3605 : *
3606 : * Updating tg's load_avg is necessary before update_cfs_share().
3607 : */
3608 : static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
3609 : {
3610 : long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
3611 :
3612 : /*
3613 : * No need to update load_avg for root_task_group as it is not used.
3614 : */
3615 : if (cfs_rq->tg == &root_task_group)
3616 : return;
3617 :
3618 : if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
3619 : atomic_long_add(delta, &cfs_rq->tg->load_avg);
3620 : cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
3621 : }
3622 : }
3623 :
3624 : /*
3625 : * Called within set_task_rq() right before setting a task's CPU. The
3626 : * caller only guarantees p->pi_lock is held; no other assumptions,
3627 : * including the state of rq->lock, should be made.
3628 : */
3629 : void set_task_rq_fair(struct sched_entity *se,
3630 : struct cfs_rq *prev, struct cfs_rq *next)
3631 : {
3632 : u64 p_last_update_time;
3633 : u64 n_last_update_time;
3634 :
3635 : if (!sched_feat(ATTACH_AGE_LOAD))
3636 : return;
3637 :
3638 : /*
3639 : * We are supposed to update the task to "current" time, then its up to
3640 : * date and ready to go to new CPU/cfs_rq. But we have difficulty in
3641 : * getting what current time is, so simply throw away the out-of-date
3642 : * time. This will result in the wakee task is less decayed, but giving
3643 : * the wakee more load sounds not bad.
3644 : */
3645 : if (!(se->avg.last_update_time && prev))
3646 : return;
3647 :
3648 : p_last_update_time = cfs_rq_last_update_time(prev);
3649 : n_last_update_time = cfs_rq_last_update_time(next);
3650 :
3651 : __update_load_avg_blocked_se(p_last_update_time, se);
3652 : se->avg.last_update_time = n_last_update_time;
3653 : }
3654 :
3655 : /*
3656 : * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
3657 : * propagate its contribution. The key to this propagation is the invariant
3658 : * that for each group:
3659 : *
3660 : * ge->avg == grq->avg (1)
3661 : *
3662 : * _IFF_ we look at the pure running and runnable sums. Because they
3663 : * represent the very same entity, just at different points in the hierarchy.
3664 : *
3665 : * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial
3666 : * and simply copies the running/runnable sum over (but still wrong, because
3667 : * the group entity and group rq do not have their PELT windows aligned).
3668 : *
3669 : * However, update_tg_cfs_load() is more complex. So we have:
3670 : *
3671 : * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2)
3672 : *
3673 : * And since, like util, the runnable part should be directly transferable,
3674 : * the following would _appear_ to be the straight forward approach:
3675 : *
3676 : * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3)
3677 : *
3678 : * And per (1) we have:
3679 : *
3680 : * ge->avg.runnable_avg == grq->avg.runnable_avg
3681 : *
3682 : * Which gives:
3683 : *
3684 : * ge->load.weight * grq->avg.load_avg
3685 : * ge->avg.load_avg = ----------------------------------- (4)
3686 : * grq->load.weight
3687 : *
3688 : * Except that is wrong!
3689 : *
3690 : * Because while for entities historical weight is not important and we
3691 : * really only care about our future and therefore can consider a pure
3692 : * runnable sum, runqueues can NOT do this.
3693 : *
3694 : * We specifically want runqueues to have a load_avg that includes
3695 : * historical weights. Those represent the blocked load, the load we expect
3696 : * to (shortly) return to us. This only works by keeping the weights as
3697 : * integral part of the sum. We therefore cannot decompose as per (3).
3698 : *
3699 : * Another reason this doesn't work is that runnable isn't a 0-sum entity.
3700 : * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the
3701 : * rq itself is runnable anywhere between 2/3 and 1 depending on how the
3702 : * runnable section of these tasks overlap (or not). If they were to perfectly
3703 : * align the rq as a whole would be runnable 2/3 of the time. If however we
3704 : * always have at least 1 runnable task, the rq as a whole is always runnable.
3705 : *
3706 : * So we'll have to approximate.. :/
3707 : *
3708 : * Given the constraint:
3709 : *
3710 : * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX
3711 : *
3712 : * We can construct a rule that adds runnable to a rq by assuming minimal
3713 : * overlap.
3714 : *
3715 : * On removal, we'll assume each task is equally runnable; which yields:
3716 : *
3717 : * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight
3718 : *
3719 : * XXX: only do this for the part of runnable > running ?
3720 : *
3721 : */
3722 : static inline void
3723 : update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3724 : {
3725 : long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg;
3726 : u32 new_sum, divider;
3727 :
3728 : /* Nothing to update */
3729 : if (!delta_avg)
3730 : return;
3731 :
3732 : /*
3733 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3734 : * See ___update_load_avg() for details.
3735 : */
3736 : divider = get_pelt_divider(&cfs_rq->avg);
3737 :
3738 :
3739 : /* Set new sched_entity's utilization */
3740 : se->avg.util_avg = gcfs_rq->avg.util_avg;
3741 : new_sum = se->avg.util_avg * divider;
3742 : delta_sum = (long)new_sum - (long)se->avg.util_sum;
3743 : se->avg.util_sum = new_sum;
3744 :
3745 : /* Update parent cfs_rq utilization */
3746 : add_positive(&cfs_rq->avg.util_avg, delta_avg);
3747 : add_positive(&cfs_rq->avg.util_sum, delta_sum);
3748 :
3749 : /* See update_cfs_rq_load_avg() */
3750 : cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
3751 : cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
3752 : }
3753 :
3754 : static inline void
3755 : update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3756 : {
3757 : long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
3758 : u32 new_sum, divider;
3759 :
3760 : /* Nothing to update */
3761 : if (!delta_avg)
3762 : return;
3763 :
3764 : /*
3765 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3766 : * See ___update_load_avg() for details.
3767 : */
3768 : divider = get_pelt_divider(&cfs_rq->avg);
3769 :
3770 : /* Set new sched_entity's runnable */
3771 : se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;
3772 : new_sum = se->avg.runnable_avg * divider;
3773 : delta_sum = (long)new_sum - (long)se->avg.runnable_sum;
3774 : se->avg.runnable_sum = new_sum;
3775 :
3776 : /* Update parent cfs_rq runnable */
3777 : add_positive(&cfs_rq->avg.runnable_avg, delta_avg);
3778 : add_positive(&cfs_rq->avg.runnable_sum, delta_sum);
3779 : /* See update_cfs_rq_load_avg() */
3780 : cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
3781 : cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
3782 : }
3783 :
3784 : static inline void
3785 : update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3786 : {
3787 : long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
3788 : unsigned long load_avg;
3789 : u64 load_sum = 0;
3790 : s64 delta_sum;
3791 : u32 divider;
3792 :
3793 : if (!runnable_sum)
3794 : return;
3795 :
3796 : gcfs_rq->prop_runnable_sum = 0;
3797 :
3798 : /*
3799 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3800 : * See ___update_load_avg() for details.
3801 : */
3802 : divider = get_pelt_divider(&cfs_rq->avg);
3803 :
3804 : if (runnable_sum >= 0) {
3805 : /*
3806 : * Add runnable; clip at LOAD_AVG_MAX. Reflects that until
3807 : * the CPU is saturated running == runnable.
3808 : */
3809 : runnable_sum += se->avg.load_sum;
3810 : runnable_sum = min_t(long, runnable_sum, divider);
3811 : } else {
3812 : /*
3813 : * Estimate the new unweighted runnable_sum of the gcfs_rq by
3814 : * assuming all tasks are equally runnable.
3815 : */
3816 : if (scale_load_down(gcfs_rq->load.weight)) {
3817 : load_sum = div_u64(gcfs_rq->avg.load_sum,
3818 : scale_load_down(gcfs_rq->load.weight));
3819 : }
3820 :
3821 : /* But make sure to not inflate se's runnable */
3822 : runnable_sum = min(se->avg.load_sum, load_sum);
3823 : }
3824 :
3825 : /*
3826 : * runnable_sum can't be lower than running_sum
3827 : * Rescale running sum to be in the same range as runnable sum
3828 : * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT]
3829 : * runnable_sum is in [0 : LOAD_AVG_MAX]
3830 : */
3831 : running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;
3832 : runnable_sum = max(runnable_sum, running_sum);
3833 :
3834 : load_sum = se_weight(se) * runnable_sum;
3835 : load_avg = div_u64(load_sum, divider);
3836 :
3837 : delta_avg = load_avg - se->avg.load_avg;
3838 : if (!delta_avg)
3839 : return;
3840 :
3841 : delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum;
3842 :
3843 : se->avg.load_sum = runnable_sum;
3844 : se->avg.load_avg = load_avg;
3845 : add_positive(&cfs_rq->avg.load_avg, delta_avg);
3846 : add_positive(&cfs_rq->avg.load_sum, delta_sum);
3847 : /* See update_cfs_rq_load_avg() */
3848 : cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
3849 : cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
3850 : }
3851 :
3852 : static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
3853 : {
3854 : cfs_rq->propagate = 1;
3855 : cfs_rq->prop_runnable_sum += runnable_sum;
3856 : }
3857 :
3858 : /* Update task and its cfs_rq load average */
3859 : static inline int propagate_entity_load_avg(struct sched_entity *se)
3860 : {
3861 : struct cfs_rq *cfs_rq, *gcfs_rq;
3862 :
3863 : if (entity_is_task(se))
3864 : return 0;
3865 :
3866 : gcfs_rq = group_cfs_rq(se);
3867 : if (!gcfs_rq->propagate)
3868 : return 0;
3869 :
3870 : gcfs_rq->propagate = 0;
3871 :
3872 : cfs_rq = cfs_rq_of(se);
3873 :
3874 : add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum);
3875 :
3876 : update_tg_cfs_util(cfs_rq, se, gcfs_rq);
3877 : update_tg_cfs_runnable(cfs_rq, se, gcfs_rq);
3878 : update_tg_cfs_load(cfs_rq, se, gcfs_rq);
3879 :
3880 : trace_pelt_cfs_tp(cfs_rq);
3881 : trace_pelt_se_tp(se);
3882 :
3883 : return 1;
3884 : }
3885 :
3886 : /*
3887 : * Check if we need to update the load and the utilization of a blocked
3888 : * group_entity:
3889 : */
3890 : static inline bool skip_blocked_update(struct sched_entity *se)
3891 : {
3892 : struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3893 :
3894 : /*
3895 : * If sched_entity still have not zero load or utilization, we have to
3896 : * decay it:
3897 : */
3898 : if (se->avg.load_avg || se->avg.util_avg)
3899 : return false;
3900 :
3901 : /*
3902 : * If there is a pending propagation, we have to update the load and
3903 : * the utilization of the sched_entity:
3904 : */
3905 : if (gcfs_rq->propagate)
3906 : return false;
3907 :
3908 : /*
3909 : * Otherwise, the load and the utilization of the sched_entity is
3910 : * already zero and there is no pending propagation, so it will be a
3911 : * waste of time to try to decay it:
3912 : */
3913 : return true;
3914 : }
3915 :
3916 : #else /* CONFIG_FAIR_GROUP_SCHED */
3917 :
3918 : static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {}
3919 :
3920 : static inline int propagate_entity_load_avg(struct sched_entity *se)
3921 : {
3922 : return 0;
3923 : }
3924 :
3925 : static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {}
3926 :
3927 : #endif /* CONFIG_FAIR_GROUP_SCHED */
3928 :
3929 : #ifdef CONFIG_NO_HZ_COMMON
3930 : static inline void migrate_se_pelt_lag(struct sched_entity *se)
3931 : {
3932 : u64 throttled = 0, now, lut;
3933 : struct cfs_rq *cfs_rq;
3934 : struct rq *rq;
3935 : bool is_idle;
3936 :
3937 : if (load_avg_is_decayed(&se->avg))
3938 : return;
3939 :
3940 : cfs_rq = cfs_rq_of(se);
3941 : rq = rq_of(cfs_rq);
3942 :
3943 : rcu_read_lock();
3944 : is_idle = is_idle_task(rcu_dereference(rq->curr));
3945 : rcu_read_unlock();
3946 :
3947 : /*
3948 : * The lag estimation comes with a cost we don't want to pay all the
3949 : * time. Hence, limiting to the case where the source CPU is idle and
3950 : * we know we are at the greatest risk to have an outdated clock.
3951 : */
3952 : if (!is_idle)
3953 : return;
3954 :
3955 : /*
3956 : * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where:
3957 : *
3958 : * last_update_time (the cfs_rq's last_update_time)
3959 : * = cfs_rq_clock_pelt()@cfs_rq_idle
3960 : * = rq_clock_pelt()@cfs_rq_idle
3961 : * - cfs->throttled_clock_pelt_time@cfs_rq_idle
3962 : *
3963 : * cfs_idle_lag (delta between rq's update and cfs_rq's update)
3964 : * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle
3965 : *
3966 : * rq_idle_lag (delta between now and rq's update)
3967 : * = sched_clock_cpu() - rq_clock()@rq_idle
3968 : *
3969 : * We can then write:
3970 : *
3971 : * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time +
3972 : * sched_clock_cpu() - rq_clock()@rq_idle
3973 : * Where:
3974 : * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle
3975 : * rq_clock()@rq_idle is rq->clock_idle
3976 : * cfs->throttled_clock_pelt_time@cfs_rq_idle
3977 : * is cfs_rq->throttled_pelt_idle
3978 : */
3979 :
3980 : #ifdef CONFIG_CFS_BANDWIDTH
3981 : throttled = u64_u32_load(cfs_rq->throttled_pelt_idle);
3982 : /* The clock has been stopped for throttling */
3983 : if (throttled == U64_MAX)
3984 : return;
3985 : #endif
3986 : now = u64_u32_load(rq->clock_pelt_idle);
3987 : /*
3988 : * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case
3989 : * is observed the old clock_pelt_idle value and the new clock_idle,
3990 : * which lead to an underestimation. The opposite would lead to an
3991 : * overestimation.
3992 : */
3993 : smp_rmb();
3994 : lut = cfs_rq_last_update_time(cfs_rq);
3995 :
3996 : now -= throttled;
3997 : if (now < lut)
3998 : /*
3999 : * cfs_rq->avg.last_update_time is more recent than our
4000 : * estimation, let's use it.
4001 : */
4002 : now = lut;
4003 : else
4004 : now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle);
4005 :
4006 : __update_load_avg_blocked_se(now, se);
4007 : }
4008 : #else
4009 : static void migrate_se_pelt_lag(struct sched_entity *se) {}
4010 : #endif
4011 :
4012 : /**
4013 : * update_cfs_rq_load_avg - update the cfs_rq's load/util averages
4014 : * @now: current time, as per cfs_rq_clock_pelt()
4015 : * @cfs_rq: cfs_rq to update
4016 : *
4017 : * The cfs_rq avg is the direct sum of all its entities (blocked and runnable)
4018 : * avg. The immediate corollary is that all (fair) tasks must be attached.
4019 : *
4020 : * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example.
4021 : *
4022 : * Return: true if the load decayed or we removed load.
4023 : *
4024 : * Since both these conditions indicate a changed cfs_rq->avg.load we should
4025 : * call update_tg_load_avg() when this function returns true.
4026 : */
4027 : static inline int
4028 : update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
4029 : {
4030 : unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0;
4031 : struct sched_avg *sa = &cfs_rq->avg;
4032 : int decayed = 0;
4033 :
4034 : if (cfs_rq->removed.nr) {
4035 : unsigned long r;
4036 : u32 divider = get_pelt_divider(&cfs_rq->avg);
4037 :
4038 : raw_spin_lock(&cfs_rq->removed.lock);
4039 : swap(cfs_rq->removed.util_avg, removed_util);
4040 : swap(cfs_rq->removed.load_avg, removed_load);
4041 : swap(cfs_rq->removed.runnable_avg, removed_runnable);
4042 : cfs_rq->removed.nr = 0;
4043 : raw_spin_unlock(&cfs_rq->removed.lock);
4044 :
4045 : r = removed_load;
4046 : sub_positive(&sa->load_avg, r);
4047 : sub_positive(&sa->load_sum, r * divider);
4048 : /* See sa->util_sum below */
4049 : sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER);
4050 :
4051 : r = removed_util;
4052 : sub_positive(&sa->util_avg, r);
4053 : sub_positive(&sa->util_sum, r * divider);
4054 : /*
4055 : * Because of rounding, se->util_sum might ends up being +1 more than
4056 : * cfs->util_sum. Although this is not a problem by itself, detaching
4057 : * a lot of tasks with the rounding problem between 2 updates of
4058 : * util_avg (~1ms) can make cfs->util_sum becoming null whereas
4059 : * cfs_util_avg is not.
4060 : * Check that util_sum is still above its lower bound for the new
4061 : * util_avg. Given that period_contrib might have moved since the last
4062 : * sync, we are only sure that util_sum must be above or equal to
4063 : * util_avg * minimum possible divider
4064 : */
4065 : sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER);
4066 :
4067 : r = removed_runnable;
4068 : sub_positive(&sa->runnable_avg, r);
4069 : sub_positive(&sa->runnable_sum, r * divider);
4070 : /* See sa->util_sum above */
4071 : sa->runnable_sum = max_t(u32, sa->runnable_sum,
4072 : sa->runnable_avg * PELT_MIN_DIVIDER);
4073 :
4074 : /*
4075 : * removed_runnable is the unweighted version of removed_load so we
4076 : * can use it to estimate removed_load_sum.
4077 : */
4078 : add_tg_cfs_propagate(cfs_rq,
4079 : -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT);
4080 :
4081 : decayed = 1;
4082 : }
4083 :
4084 : decayed |= __update_load_avg_cfs_rq(now, cfs_rq);
4085 : u64_u32_store_copy(sa->last_update_time,
4086 : cfs_rq->last_update_time_copy,
4087 : sa->last_update_time);
4088 : return decayed;
4089 : }
4090 :
4091 : /**
4092 : * attach_entity_load_avg - attach this entity to its cfs_rq load avg
4093 : * @cfs_rq: cfs_rq to attach to
4094 : * @se: sched_entity to attach
4095 : *
4096 : * Must call update_cfs_rq_load_avg() before this, since we rely on
4097 : * cfs_rq->avg.last_update_time being current.
4098 : */
4099 : static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
4100 : {
4101 : /*
4102 : * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
4103 : * See ___update_load_avg() for details.
4104 : */
4105 : u32 divider = get_pelt_divider(&cfs_rq->avg);
4106 :
4107 : /*
4108 : * When we attach the @se to the @cfs_rq, we must align the decay
4109 : * window because without that, really weird and wonderful things can
4110 : * happen.
4111 : *
4112 : * XXX illustrate
4113 : */
4114 : se->avg.last_update_time = cfs_rq->avg.last_update_time;
4115 : se->avg.period_contrib = cfs_rq->avg.period_contrib;
4116 :
4117 : /*
4118 : * Hell(o) Nasty stuff.. we need to recompute _sum based on the new
4119 : * period_contrib. This isn't strictly correct, but since we're
4120 : * entirely outside of the PELT hierarchy, nobody cares if we truncate
4121 : * _sum a little.
4122 : */
4123 : se->avg.util_sum = se->avg.util_avg * divider;
4124 :
4125 : se->avg.runnable_sum = se->avg.runnable_avg * divider;
4126 :
4127 : se->avg.load_sum = se->avg.load_avg * divider;
4128 : if (se_weight(se) < se->avg.load_sum)
4129 : se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se));
4130 : else
4131 : se->avg.load_sum = 1;
4132 :
4133 : enqueue_load_avg(cfs_rq, se);
4134 : cfs_rq->avg.util_avg += se->avg.util_avg;
4135 : cfs_rq->avg.util_sum += se->avg.util_sum;
4136 : cfs_rq->avg.runnable_avg += se->avg.runnable_avg;
4137 : cfs_rq->avg.runnable_sum += se->avg.runnable_sum;
4138 :
4139 : add_tg_cfs_propagate(cfs_rq, se->avg.load_sum);
4140 :
4141 : cfs_rq_util_change(cfs_rq, 0);
4142 :
4143 : trace_pelt_cfs_tp(cfs_rq);
4144 : }
4145 :
4146 : /**
4147 : * detach_entity_load_avg - detach this entity from its cfs_rq load avg
4148 : * @cfs_rq: cfs_rq to detach from
4149 : * @se: sched_entity to detach
4150 : *
4151 : * Must call update_cfs_rq_load_avg() before this, since we rely on
4152 : * cfs_rq->avg.last_update_time being current.
4153 : */
4154 : static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
4155 : {
4156 : dequeue_load_avg(cfs_rq, se);
4157 : sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
4158 : sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);
4159 : /* See update_cfs_rq_load_avg() */
4160 : cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
4161 : cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
4162 :
4163 : sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg);
4164 : sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum);
4165 : /* See update_cfs_rq_load_avg() */
4166 : cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
4167 : cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
4168 :
4169 : add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);
4170 :
4171 : cfs_rq_util_change(cfs_rq, 0);
4172 :
4173 : trace_pelt_cfs_tp(cfs_rq);
4174 : }
4175 :
4176 : /*
4177 : * Optional action to be done while updating the load average
4178 : */
4179 : #define UPDATE_TG 0x1
4180 : #define SKIP_AGE_LOAD 0x2
4181 : #define DO_ATTACH 0x4
4182 : #define DO_DETACH 0x8
4183 :
4184 : /* Update task and its cfs_rq load average */
4185 : static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4186 : {
4187 : u64 now = cfs_rq_clock_pelt(cfs_rq);
4188 : int decayed;
4189 :
4190 : /*
4191 : * Track task load average for carrying it to new CPU after migrated, and
4192 : * track group sched_entity load average for task_h_load calc in migration
4193 : */
4194 : if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))
4195 : __update_load_avg_se(now, cfs_rq, se);
4196 :
4197 : decayed = update_cfs_rq_load_avg(now, cfs_rq);
4198 : decayed |= propagate_entity_load_avg(se);
4199 :
4200 : if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
4201 :
4202 : /*
4203 : * DO_ATTACH means we're here from enqueue_entity().
4204 : * !last_update_time means we've passed through
4205 : * migrate_task_rq_fair() indicating we migrated.
4206 : *
4207 : * IOW we're enqueueing a task on a new CPU.
4208 : */
4209 : attach_entity_load_avg(cfs_rq, se);
4210 : update_tg_load_avg(cfs_rq);
4211 :
4212 : } else if (flags & DO_DETACH) {
4213 : /*
4214 : * DO_DETACH means we're here from dequeue_entity()
4215 : * and we are migrating task out of the CPU.
4216 : */
4217 : detach_entity_load_avg(cfs_rq, se);
4218 : update_tg_load_avg(cfs_rq);
4219 : } else if (decayed) {
4220 : cfs_rq_util_change(cfs_rq, 0);
4221 :
4222 : if (flags & UPDATE_TG)
4223 : update_tg_load_avg(cfs_rq);
4224 : }
4225 : }
4226 :
4227 : /*
4228 : * Synchronize entity load avg of dequeued entity without locking
4229 : * the previous rq.
4230 : */
4231 : static void sync_entity_load_avg(struct sched_entity *se)
4232 : {
4233 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4234 : u64 last_update_time;
4235 :
4236 : last_update_time = cfs_rq_last_update_time(cfs_rq);
4237 : __update_load_avg_blocked_se(last_update_time, se);
4238 : }
4239 :
4240 : /*
4241 : * Task first catches up with cfs_rq, and then subtract
4242 : * itself from the cfs_rq (task must be off the queue now).
4243 : */
4244 : static void remove_entity_load_avg(struct sched_entity *se)
4245 : {
4246 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4247 : unsigned long flags;
4248 :
4249 : /*
4250 : * tasks cannot exit without having gone through wake_up_new_task() ->
4251 : * enqueue_task_fair() which will have added things to the cfs_rq,
4252 : * so we can remove unconditionally.
4253 : */
4254 :
4255 : sync_entity_load_avg(se);
4256 :
4257 : raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags);
4258 : ++cfs_rq->removed.nr;
4259 : cfs_rq->removed.util_avg += se->avg.util_avg;
4260 : cfs_rq->removed.load_avg += se->avg.load_avg;
4261 : cfs_rq->removed.runnable_avg += se->avg.runnable_avg;
4262 : raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags);
4263 : }
4264 :
4265 : static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq)
4266 : {
4267 : return cfs_rq->avg.runnable_avg;
4268 : }
4269 :
4270 : static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
4271 : {
4272 : return cfs_rq->avg.load_avg;
4273 : }
4274 :
4275 : static int newidle_balance(struct rq *this_rq, struct rq_flags *rf);
4276 :
4277 : static inline unsigned long task_util(struct task_struct *p)
4278 : {
4279 : return READ_ONCE(p->se.avg.util_avg);
4280 : }
4281 :
4282 : static inline unsigned long _task_util_est(struct task_struct *p)
4283 : {
4284 : struct util_est ue = READ_ONCE(p->se.avg.util_est);
4285 :
4286 : return max(ue.ewma, (ue.enqueued & ~UTIL_AVG_UNCHANGED));
4287 : }
4288 :
4289 : static inline unsigned long task_util_est(struct task_struct *p)
4290 : {
4291 : return max(task_util(p), _task_util_est(p));
4292 : }
4293 :
4294 : #ifdef CONFIG_UCLAMP_TASK
4295 : static inline unsigned long uclamp_task_util(struct task_struct *p,
4296 : unsigned long uclamp_min,
4297 : unsigned long uclamp_max)
4298 : {
4299 : return clamp(task_util_est(p), uclamp_min, uclamp_max);
4300 : }
4301 : #else
4302 : static inline unsigned long uclamp_task_util(struct task_struct *p,
4303 : unsigned long uclamp_min,
4304 : unsigned long uclamp_max)
4305 : {
4306 : return task_util_est(p);
4307 : }
4308 : #endif
4309 :
4310 : static inline void util_est_enqueue(struct cfs_rq *cfs_rq,
4311 : struct task_struct *p)
4312 : {
4313 : unsigned int enqueued;
4314 :
4315 : if (!sched_feat(UTIL_EST))
4316 : return;
4317 :
4318 : /* Update root cfs_rq's estimated utilization */
4319 : enqueued = cfs_rq->avg.util_est.enqueued;
4320 : enqueued += _task_util_est(p);
4321 : WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
4322 :
4323 : trace_sched_util_est_cfs_tp(cfs_rq);
4324 : }
4325 :
4326 : static inline void util_est_dequeue(struct cfs_rq *cfs_rq,
4327 : struct task_struct *p)
4328 : {
4329 : unsigned int enqueued;
4330 :
4331 : if (!sched_feat(UTIL_EST))
4332 : return;
4333 :
4334 : /* Update root cfs_rq's estimated utilization */
4335 : enqueued = cfs_rq->avg.util_est.enqueued;
4336 : enqueued -= min_t(unsigned int, enqueued, _task_util_est(p));
4337 : WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued);
4338 :
4339 : trace_sched_util_est_cfs_tp(cfs_rq);
4340 : }
4341 :
4342 : #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
4343 :
4344 : /*
4345 : * Check if a (signed) value is within a specified (unsigned) margin,
4346 : * based on the observation that:
4347 : *
4348 : * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1)
4349 : *
4350 : * NOTE: this only works when value + margin < INT_MAX.
4351 : */
4352 : static inline bool within_margin(int value, int margin)
4353 : {
4354 : return ((unsigned int)(value + margin - 1) < (2 * margin - 1));
4355 : }
4356 :
4357 : static inline void util_est_update(struct cfs_rq *cfs_rq,
4358 : struct task_struct *p,
4359 : bool task_sleep)
4360 : {
4361 : long last_ewma_diff, last_enqueued_diff;
4362 : struct util_est ue;
4363 :
4364 : if (!sched_feat(UTIL_EST))
4365 : return;
4366 :
4367 : /*
4368 : * Skip update of task's estimated utilization when the task has not
4369 : * yet completed an activation, e.g. being migrated.
4370 : */
4371 : if (!task_sleep)
4372 : return;
4373 :
4374 : /*
4375 : * If the PELT values haven't changed since enqueue time,
4376 : * skip the util_est update.
4377 : */
4378 : ue = p->se.avg.util_est;
4379 : if (ue.enqueued & UTIL_AVG_UNCHANGED)
4380 : return;
4381 :
4382 : last_enqueued_diff = ue.enqueued;
4383 :
4384 : /*
4385 : * Reset EWMA on utilization increases, the moving average is used only
4386 : * to smooth utilization decreases.
4387 : */
4388 : ue.enqueued = task_util(p);
4389 : if (sched_feat(UTIL_EST_FASTUP)) {
4390 : if (ue.ewma < ue.enqueued) {
4391 : ue.ewma = ue.enqueued;
4392 : goto done;
4393 : }
4394 : }
4395 :
4396 : /*
4397 : * Skip update of task's estimated utilization when its members are
4398 : * already ~1% close to its last activation value.
4399 : */
4400 : last_ewma_diff = ue.enqueued - ue.ewma;
4401 : last_enqueued_diff -= ue.enqueued;
4402 : if (within_margin(last_ewma_diff, UTIL_EST_MARGIN)) {
4403 : if (!within_margin(last_enqueued_diff, UTIL_EST_MARGIN))
4404 : goto done;
4405 :
4406 : return;
4407 : }
4408 :
4409 : /*
4410 : * To avoid overestimation of actual task utilization, skip updates if
4411 : * we cannot grant there is idle time in this CPU.
4412 : */
4413 : if (task_util(p) > capacity_orig_of(cpu_of(rq_of(cfs_rq))))
4414 : return;
4415 :
4416 : /*
4417 : * Update Task's estimated utilization
4418 : *
4419 : * When *p completes an activation we can consolidate another sample
4420 : * of the task size. This is done by storing the current PELT value
4421 : * as ue.enqueued and by using this value to update the Exponential
4422 : * Weighted Moving Average (EWMA):
4423 : *
4424 : * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1)
4425 : * = w * task_util(p) + ewma(t-1) - w * ewma(t-1)
4426 : * = w * (task_util(p) - ewma(t-1)) + ewma(t-1)
4427 : * = w * ( last_ewma_diff ) + ewma(t-1)
4428 : * = w * (last_ewma_diff + ewma(t-1) / w)
4429 : *
4430 : * Where 'w' is the weight of new samples, which is configured to be
4431 : * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
4432 : */
4433 : ue.ewma <<= UTIL_EST_WEIGHT_SHIFT;
4434 : ue.ewma += last_ewma_diff;
4435 : ue.ewma >>= UTIL_EST_WEIGHT_SHIFT;
4436 : done:
4437 : ue.enqueued |= UTIL_AVG_UNCHANGED;
4438 : WRITE_ONCE(p->se.avg.util_est, ue);
4439 :
4440 : trace_sched_util_est_se_tp(&p->se);
4441 : }
4442 :
4443 : static inline int util_fits_cpu(unsigned long util,
4444 : unsigned long uclamp_min,
4445 : unsigned long uclamp_max,
4446 : int cpu)
4447 : {
4448 : unsigned long capacity_orig, capacity_orig_thermal;
4449 : unsigned long capacity = capacity_of(cpu);
4450 : bool fits, uclamp_max_fits;
4451 :
4452 : /*
4453 : * Check if the real util fits without any uclamp boost/cap applied.
4454 : */
4455 : fits = fits_capacity(util, capacity);
4456 :
4457 : if (!uclamp_is_used())
4458 : return fits;
4459 :
4460 : /*
4461 : * We must use capacity_orig_of() for comparing against uclamp_min and
4462 : * uclamp_max. We only care about capacity pressure (by using
4463 : * capacity_of()) for comparing against the real util.
4464 : *
4465 : * If a task is boosted to 1024 for example, we don't want a tiny
4466 : * pressure to skew the check whether it fits a CPU or not.
4467 : *
4468 : * Similarly if a task is capped to capacity_orig_of(little_cpu), it
4469 : * should fit a little cpu even if there's some pressure.
4470 : *
4471 : * Only exception is for thermal pressure since it has a direct impact
4472 : * on available OPP of the system.
4473 : *
4474 : * We honour it for uclamp_min only as a drop in performance level
4475 : * could result in not getting the requested minimum performance level.
4476 : *
4477 : * For uclamp_max, we can tolerate a drop in performance level as the
4478 : * goal is to cap the task. So it's okay if it's getting less.
4479 : */
4480 : capacity_orig = capacity_orig_of(cpu);
4481 : capacity_orig_thermal = capacity_orig - arch_scale_thermal_pressure(cpu);
4482 :
4483 : /*
4484 : * We want to force a task to fit a cpu as implied by uclamp_max.
4485 : * But we do have some corner cases to cater for..
4486 : *
4487 : *
4488 : * C=z
4489 : * | ___
4490 : * | C=y | |
4491 : * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max
4492 : * | C=x | | | |
4493 : * | ___ | | | |
4494 : * | | | | | | | (util somewhere in this region)
4495 : * | | | | | | |
4496 : * | | | | | | |
4497 : * +----------------------------------------
4498 : * cpu0 cpu1 cpu2
4499 : *
4500 : * In the above example if a task is capped to a specific performance
4501 : * point, y, then when:
4502 : *
4503 : * * util = 80% of x then it does not fit on cpu0 and should migrate
4504 : * to cpu1
4505 : * * util = 80% of y then it is forced to fit on cpu1 to honour
4506 : * uclamp_max request.
4507 : *
4508 : * which is what we're enforcing here. A task always fits if
4509 : * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig,
4510 : * the normal upmigration rules should withhold still.
4511 : *
4512 : * Only exception is when we are on max capacity, then we need to be
4513 : * careful not to block overutilized state. This is so because:
4514 : *
4515 : * 1. There's no concept of capping at max_capacity! We can't go
4516 : * beyond this performance level anyway.
4517 : * 2. The system is being saturated when we're operating near
4518 : * max capacity, it doesn't make sense to block overutilized.
4519 : */
4520 : uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE);
4521 : uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig);
4522 : fits = fits || uclamp_max_fits;
4523 :
4524 : /*
4525 : *
4526 : * C=z
4527 : * | ___ (region a, capped, util >= uclamp_max)
4528 : * | C=y | |
4529 : * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max
4530 : * | C=x | | | |
4531 : * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max)
4532 : * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min
4533 : * | | | | | | |
4534 : * | | | | | | | (region c, boosted, util < uclamp_min)
4535 : * +----------------------------------------
4536 : * cpu0 cpu1 cpu2
4537 : *
4538 : * a) If util > uclamp_max, then we're capped, we don't care about
4539 : * actual fitness value here. We only care if uclamp_max fits
4540 : * capacity without taking margin/pressure into account.
4541 : * See comment above.
4542 : *
4543 : * b) If uclamp_min <= util <= uclamp_max, then the normal
4544 : * fits_capacity() rules apply. Except we need to ensure that we
4545 : * enforce we remain within uclamp_max, see comment above.
4546 : *
4547 : * c) If util < uclamp_min, then we are boosted. Same as (b) but we
4548 : * need to take into account the boosted value fits the CPU without
4549 : * taking margin/pressure into account.
4550 : *
4551 : * Cases (a) and (b) are handled in the 'fits' variable already. We
4552 : * just need to consider an extra check for case (c) after ensuring we
4553 : * handle the case uclamp_min > uclamp_max.
4554 : */
4555 : uclamp_min = min(uclamp_min, uclamp_max);
4556 : if (fits && (util < uclamp_min) && (uclamp_min > capacity_orig_thermal))
4557 : return -1;
4558 :
4559 : return fits;
4560 : }
4561 :
4562 : static inline int task_fits_cpu(struct task_struct *p, int cpu)
4563 : {
4564 : unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN);
4565 : unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX);
4566 : unsigned long util = task_util_est(p);
4567 : /*
4568 : * Return true only if the cpu fully fits the task requirements, which
4569 : * include the utilization but also the performance hints.
4570 : */
4571 : return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0);
4572 : }
4573 :
4574 : static inline void update_misfit_status(struct task_struct *p, struct rq *rq)
4575 : {
4576 : if (!sched_asym_cpucap_active())
4577 : return;
4578 :
4579 : if (!p || p->nr_cpus_allowed == 1) {
4580 : rq->misfit_task_load = 0;
4581 : return;
4582 : }
4583 :
4584 : if (task_fits_cpu(p, cpu_of(rq))) {
4585 : rq->misfit_task_load = 0;
4586 : return;
4587 : }
4588 :
4589 : /*
4590 : * Make sure that misfit_task_load will not be null even if
4591 : * task_h_load() returns 0.
4592 : */
4593 : rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1);
4594 : }
4595 :
4596 : #else /* CONFIG_SMP */
4597 :
4598 : static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
4599 : {
4600 : return true;
4601 : }
4602 :
4603 : #define UPDATE_TG 0x0
4604 : #define SKIP_AGE_LOAD 0x0
4605 : #define DO_ATTACH 0x0
4606 : #define DO_DETACH 0x0
4607 :
4608 : static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1)
4609 : {
4610 6981 : cfs_rq_util_change(cfs_rq, 0);
4611 : }
4612 :
4613 : static inline void remove_entity_load_avg(struct sched_entity *se) {}
4614 :
4615 : static inline void
4616 : attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
4617 : static inline void
4618 : detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
4619 :
4620 : static inline int newidle_balance(struct rq *rq, struct rq_flags *rf)
4621 : {
4622 : return 0;
4623 : }
4624 :
4625 : static inline void
4626 : util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
4627 :
4628 : static inline void
4629 : util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
4630 :
4631 : static inline void
4632 : util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p,
4633 : bool task_sleep) {}
4634 : static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {}
4635 :
4636 : #endif /* CONFIG_SMP */
4637 :
4638 : static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
4639 : {
4640 : #ifdef CONFIG_SCHED_DEBUG
4641 4344 : s64 d = se->vruntime - cfs_rq->min_vruntime;
4642 :
4643 : if (d < 0)
4644 : d = -d;
4645 :
4646 : if (d > 3*sysctl_sched_latency)
4647 : schedstat_inc(cfs_rq->nr_spread_over);
4648 : #endif
4649 : }
4650 :
4651 : static void
4652 2128 : place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
4653 : {
4654 2128 : u64 vruntime = cfs_rq->min_vruntime;
4655 : u64 sleep_time;
4656 :
4657 : /*
4658 : * The 'current' period is already promised to the current tasks,
4659 : * however the extra weight of the new task will slow them down a
4660 : * little, place the new task so that it fits in the slot that
4661 : * stays open at the end.
4662 : */
4663 2128 : if (initial && sched_feat(START_DEBIT))
4664 340 : vruntime += sched_vslice(cfs_rq, se);
4665 :
4666 : /* sleeps up to a single latency don't count. */
4667 2128 : if (!initial) {
4668 : unsigned long thresh;
4669 :
4670 1788 : if (se_is_idle(se))
4671 : thresh = sysctl_sched_min_granularity;
4672 : else
4673 1788 : thresh = sysctl_sched_latency;
4674 :
4675 : /*
4676 : * Halve their sleep time's effect, to allow
4677 : * for a gentler effect of sleepers:
4678 : */
4679 1788 : if (sched_feat(GENTLE_FAIR_SLEEPERS))
4680 1788 : thresh >>= 1;
4681 :
4682 1788 : vruntime -= thresh;
4683 : }
4684 :
4685 : /*
4686 : * Pull vruntime of the entity being placed to the base level of
4687 : * cfs_rq, to prevent boosting it if placed backwards. If the entity
4688 : * slept for a long time, don't even try to compare its vruntime with
4689 : * the base as it may be too far off and the comparison may get
4690 : * inversed due to s64 overflow.
4691 : */
4692 4256 : sleep_time = rq_clock_task(rq_of(cfs_rq)) - se->exec_start;
4693 2128 : if ((s64)sleep_time > 60LL * NSEC_PER_SEC)
4694 0 : se->vruntime = vruntime;
4695 : else
4696 4256 : se->vruntime = max_vruntime(se->vruntime, vruntime);
4697 2128 : }
4698 :
4699 : static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
4700 :
4701 : static inline bool cfs_bandwidth_used(void);
4702 :
4703 : /*
4704 : * MIGRATION
4705 : *
4706 : * dequeue
4707 : * update_curr()
4708 : * update_min_vruntime()
4709 : * vruntime -= min_vruntime
4710 : *
4711 : * enqueue
4712 : * update_curr()
4713 : * update_min_vruntime()
4714 : * vruntime += min_vruntime
4715 : *
4716 : * this way the vruntime transition between RQs is done when both
4717 : * min_vruntime are up-to-date.
4718 : *
4719 : * WAKEUP (remote)
4720 : *
4721 : * ->migrate_task_rq_fair() (p->state == TASK_WAKING)
4722 : * vruntime -= min_vruntime
4723 : *
4724 : * enqueue
4725 : * update_curr()
4726 : * update_min_vruntime()
4727 : * vruntime += min_vruntime
4728 : *
4729 : * this way we don't have the most up-to-date min_vruntime on the originating
4730 : * CPU and an up-to-date min_vruntime on the destination CPU.
4731 : */
4732 :
4733 : static void
4734 2132 : enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4735 : {
4736 2132 : bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED);
4737 2132 : bool curr = cfs_rq->curr == se;
4738 :
4739 : /*
4740 : * If we're the current task, we must renormalise before calling
4741 : * update_curr().
4742 : */
4743 2132 : if (renorm && curr)
4744 0 : se->vruntime += cfs_rq->min_vruntime;
4745 :
4746 2132 : update_curr(cfs_rq);
4747 :
4748 : /*
4749 : * Otherwise, renormalise after, such that we're placed at the current
4750 : * moment in time, instead of some random moment in the past. Being
4751 : * placed in the past could significantly boost this task to the
4752 : * fairness detriment of existing tasks.
4753 : */
4754 2132 : if (renorm && !curr)
4755 344 : se->vruntime += cfs_rq->min_vruntime;
4756 :
4757 : /*
4758 : * When enqueuing a sched_entity, we must:
4759 : * - Update loads to have both entity and cfs_rq synced with now.
4760 : * - For group_entity, update its runnable_weight to reflect the new
4761 : * h_nr_running of its group cfs_rq.
4762 : * - For group_entity, update its weight to reflect the new share of
4763 : * its group cfs_rq
4764 : * - Add its new weight to cfs_rq->load.weight
4765 : */
4766 2132 : update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
4767 2132 : se_update_runnable(se);
4768 2132 : update_cfs_group(se);
4769 4264 : account_entity_enqueue(cfs_rq, se);
4770 :
4771 2132 : if (flags & ENQUEUE_WAKEUP)
4772 1788 : place_entity(cfs_rq, se, 0);
4773 :
4774 : check_schedstat_required();
4775 2132 : update_stats_enqueue_fair(cfs_rq, se, flags);
4776 2132 : check_spread(cfs_rq, se);
4777 2132 : if (!curr)
4778 2132 : __enqueue_entity(cfs_rq, se);
4779 2132 : se->on_rq = 1;
4780 :
4781 : if (cfs_rq->nr_running == 1) {
4782 : check_enqueue_throttle(cfs_rq);
4783 : if (!throttled_hierarchy(cfs_rq))
4784 : list_add_leaf_cfs_rq(cfs_rq);
4785 : }
4786 2132 : }
4787 :
4788 : static void __clear_buddies_last(struct sched_entity *se)
4789 : {
4790 0 : for_each_sched_entity(se) {
4791 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4792 0 : if (cfs_rq->last != se)
4793 : break;
4794 :
4795 0 : cfs_rq->last = NULL;
4796 : }
4797 : }
4798 :
4799 : static void __clear_buddies_next(struct sched_entity *se)
4800 : {
4801 746 : for_each_sched_entity(se) {
4802 1492 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4803 746 : if (cfs_rq->next != se)
4804 : break;
4805 :
4806 746 : cfs_rq->next = NULL;
4807 : }
4808 : }
4809 :
4810 : static void __clear_buddies_skip(struct sched_entity *se)
4811 : {
4812 0 : for_each_sched_entity(se) {
4813 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
4814 0 : if (cfs_rq->skip != se)
4815 : break;
4816 :
4817 0 : cfs_rq->skip = NULL;
4818 : }
4819 : }
4820 :
4821 4468 : static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
4822 : {
4823 4468 : if (cfs_rq->last == se)
4824 : __clear_buddies_last(se);
4825 :
4826 4468 : if (cfs_rq->next == se)
4827 : __clear_buddies_next(se);
4828 :
4829 4468 : if (cfs_rq->skip == se)
4830 : __clear_buddies_skip(se);
4831 4468 : }
4832 :
4833 : static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
4834 :
4835 : static void
4836 2130 : dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4837 : {
4838 2130 : int action = UPDATE_TG;
4839 :
4840 4260 : if (entity_is_task(se) && task_on_rq_migrating(task_of(se)))
4841 : action |= DO_DETACH;
4842 :
4843 : /*
4844 : * Update run-time statistics of the 'current'.
4845 : */
4846 2130 : update_curr(cfs_rq);
4847 :
4848 : /*
4849 : * When dequeuing a sched_entity, we must:
4850 : * - Update loads to have both entity and cfs_rq synced with now.
4851 : * - For group_entity, update its runnable_weight to reflect the new
4852 : * h_nr_running of its group cfs_rq.
4853 : * - Subtract its previous weight from cfs_rq->load.weight.
4854 : * - For group entity, update its weight to reflect the new share
4855 : * of its group cfs_rq.
4856 : */
4857 2130 : update_load_avg(cfs_rq, se, action);
4858 2130 : se_update_runnable(se);
4859 :
4860 2130 : update_stats_dequeue_fair(cfs_rq, se, flags);
4861 :
4862 2130 : clear_buddies(cfs_rq, se);
4863 :
4864 2130 : if (se != cfs_rq->curr)
4865 : __dequeue_entity(cfs_rq, se);
4866 2130 : se->on_rq = 0;
4867 4260 : account_entity_dequeue(cfs_rq, se);
4868 :
4869 : /*
4870 : * Normalize after update_curr(); which will also have moved
4871 : * min_vruntime if @se is the one holding it back. But before doing
4872 : * update_min_vruntime() again, which will discount @se's position and
4873 : * can move min_vruntime forward still more.
4874 : */
4875 2130 : if (!(flags & DEQUEUE_SLEEP))
4876 4 : se->vruntime -= cfs_rq->min_vruntime;
4877 :
4878 : /* return excess runtime on last dequeue */
4879 2130 : return_cfs_rq_runtime(cfs_rq);
4880 :
4881 2130 : update_cfs_group(se);
4882 :
4883 : /*
4884 : * Now advance min_vruntime if @se was the entity holding it back,
4885 : * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be
4886 : * put back on, and if we advance min_vruntime, we'll be placed back
4887 : * further than we started -- ie. we'll be penalized.
4888 : */
4889 2130 : if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE)
4890 2126 : update_min_vruntime(cfs_rq);
4891 :
4892 : if (cfs_rq->nr_running == 0)
4893 : update_idle_cfs_rq_clock_pelt(cfs_rq);
4894 2130 : }
4895 :
4896 : /*
4897 : * Preempt the current task with a newly woken task if needed:
4898 : */
4899 : static void
4900 125 : check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4901 : {
4902 : unsigned long ideal_runtime, delta_exec;
4903 : struct sched_entity *se;
4904 : s64 delta;
4905 :
4906 : /*
4907 : * When many tasks blow up the sched_period; it is possible that
4908 : * sched_slice() reports unusually large results (when many tasks are
4909 : * very light for example). Therefore impose a maximum.
4910 : */
4911 125 : ideal_runtime = min_t(u64, sched_slice(cfs_rq, curr), sysctl_sched_latency);
4912 :
4913 125 : delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
4914 125 : if (delta_exec > ideal_runtime) {
4915 125 : resched_curr(rq_of(cfs_rq));
4916 : /*
4917 : * The current task ran long enough, ensure it doesn't get
4918 : * re-elected due to buddy favours.
4919 : */
4920 125 : clear_buddies(cfs_rq, curr);
4921 125 : return;
4922 : }
4923 :
4924 : /*
4925 : * Ensure that a task that missed wakeup preemption by a
4926 : * narrow margin doesn't have to wait for a full slice.
4927 : * This also mitigates buddy induced latencies under load.
4928 : */
4929 0 : if (delta_exec < sysctl_sched_min_granularity)
4930 : return;
4931 :
4932 0 : se = __pick_first_entity(cfs_rq);
4933 0 : delta = curr->vruntime - se->vruntime;
4934 :
4935 0 : if (delta < 0)
4936 : return;
4937 :
4938 0 : if (delta > ideal_runtime)
4939 0 : resched_curr(rq_of(cfs_rq));
4940 : }
4941 :
4942 : static void
4943 2213 : set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
4944 : {
4945 2213 : clear_buddies(cfs_rq, se);
4946 :
4947 : /* 'current' is not kept within the tree. */
4948 2213 : if (se->on_rq) {
4949 : /*
4950 : * Any task has to be enqueued before it get to execute on
4951 : * a CPU. So account for the time it spent waiting on the
4952 : * runqueue.
4953 : */
4954 4426 : update_stats_wait_end_fair(cfs_rq, se);
4955 : __dequeue_entity(cfs_rq, se);
4956 : update_load_avg(cfs_rq, se, UPDATE_TG);
4957 : }
4958 :
4959 4426 : update_stats_curr_start(cfs_rq, se);
4960 2213 : cfs_rq->curr = se;
4961 :
4962 : /*
4963 : * Track our maximum slice length, if the CPU's load is at
4964 : * least twice that of our own weight (i.e. dont track it
4965 : * when there are only lesser-weight tasks around):
4966 : */
4967 : if (schedstat_enabled() &&
4968 : rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) {
4969 : struct sched_statistics *stats;
4970 :
4971 : stats = __schedstats_from_se(se);
4972 : __schedstat_set(stats->slice_max,
4973 : max((u64)stats->slice_max,
4974 : se->sum_exec_runtime - se->prev_sum_exec_runtime));
4975 : }
4976 :
4977 2213 : se->prev_sum_exec_runtime = se->sum_exec_runtime;
4978 2213 : }
4979 :
4980 : static int
4981 : wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
4982 :
4983 : /*
4984 : * Pick the next process, keeping these things in mind, in this order:
4985 : * 1) keep things fair between processes/task groups
4986 : * 2) pick the "next" process, since someone really wants that to run
4987 : * 3) pick the "last" process, for cache locality
4988 : * 4) do not run the "skip" process, if something else is available
4989 : */
4990 : static struct sched_entity *
4991 2209 : pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr)
4992 : {
4993 2209 : struct sched_entity *left = __pick_first_entity(cfs_rq);
4994 : struct sched_entity *se;
4995 :
4996 : /*
4997 : * If curr is set we have to see if its left of the leftmost entity
4998 : * still in the tree, provided there was anything in the tree at all.
4999 : */
5000 2209 : if (!left || (curr && entity_before(curr, left)))
5001 : left = curr;
5002 :
5003 2209 : se = left; /* ideally we run the leftmost entity */
5004 :
5005 : /*
5006 : * Avoid running the skip buddy, if running something else can
5007 : * be done without getting too unfair.
5008 : */
5009 2209 : if (cfs_rq->skip && cfs_rq->skip == se) {
5010 : struct sched_entity *second;
5011 :
5012 0 : if (se == curr) {
5013 : second = __pick_first_entity(cfs_rq);
5014 : } else {
5015 0 : second = __pick_next_entity(se);
5016 0 : if (!second || (curr && entity_before(curr, second)))
5017 : second = curr;
5018 : }
5019 :
5020 0 : if (second && wakeup_preempt_entity(second, left) < 1)
5021 0 : se = second;
5022 : }
5023 :
5024 2209 : if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) {
5025 : /*
5026 : * Someone really wants this to run. If it's not unfair, run it.
5027 : */
5028 746 : se = cfs_rq->next;
5029 1463 : } else if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) {
5030 : /*
5031 : * Prefer last buddy, try to return the CPU to a preempted task.
5032 : */
5033 0 : se = cfs_rq->last;
5034 : }
5035 :
5036 2209 : return se;
5037 : }
5038 :
5039 : static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5040 :
5041 2212 : static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
5042 : {
5043 : /*
5044 : * If still on the runqueue then deactivate_task()
5045 : * was not called and update_curr() has to be done:
5046 : */
5047 2212 : if (prev->on_rq)
5048 82 : update_curr(cfs_rq);
5049 :
5050 : /* throttle cfs_rqs exceeding runtime */
5051 2212 : check_cfs_rq_runtime(cfs_rq);
5052 :
5053 2212 : check_spread(cfs_rq, prev);
5054 :
5055 2212 : if (prev->on_rq) {
5056 82 : update_stats_wait_start_fair(cfs_rq, prev);
5057 : /* Put 'current' back into the tree. */
5058 82 : __enqueue_entity(cfs_rq, prev);
5059 : /* in !on_rq case, update occurred at dequeue */
5060 82 : update_load_avg(cfs_rq, prev, 0);
5061 : }
5062 2212 : cfs_rq->curr = NULL;
5063 2212 : }
5064 :
5065 : static void
5066 2719 : entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
5067 : {
5068 : /*
5069 : * Update run-time statistics of the 'current'.
5070 : */
5071 2719 : update_curr(cfs_rq);
5072 :
5073 : /*
5074 : * Ensure that runnable average is periodically updated.
5075 : */
5076 2719 : update_load_avg(cfs_rq, curr, UPDATE_TG);
5077 2719 : update_cfs_group(curr);
5078 :
5079 : #ifdef CONFIG_SCHED_HRTICK
5080 : /*
5081 : * queued ticks are scheduled to match the slice, so don't bother
5082 : * validating it and just reschedule.
5083 : */
5084 : if (queued) {
5085 : resched_curr(rq_of(cfs_rq));
5086 : return;
5087 : }
5088 : /*
5089 : * don't let the period tick interfere with the hrtick preemption
5090 : */
5091 : if (!sched_feat(DOUBLE_TICK) &&
5092 : hrtimer_active(&rq_of(cfs_rq)->hrtick_timer))
5093 : return;
5094 : #endif
5095 :
5096 2719 : if (cfs_rq->nr_running > 1)
5097 125 : check_preempt_tick(cfs_rq, curr);
5098 2719 : }
5099 :
5100 :
5101 : /**************************************************
5102 : * CFS bandwidth control machinery
5103 : */
5104 :
5105 : #ifdef CONFIG_CFS_BANDWIDTH
5106 :
5107 : #ifdef CONFIG_JUMP_LABEL
5108 : static struct static_key __cfs_bandwidth_used;
5109 :
5110 : static inline bool cfs_bandwidth_used(void)
5111 : {
5112 : return static_key_false(&__cfs_bandwidth_used);
5113 : }
5114 :
5115 : void cfs_bandwidth_usage_inc(void)
5116 : {
5117 : static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used);
5118 : }
5119 :
5120 : void cfs_bandwidth_usage_dec(void)
5121 : {
5122 : static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used);
5123 : }
5124 : #else /* CONFIG_JUMP_LABEL */
5125 : static bool cfs_bandwidth_used(void)
5126 : {
5127 : return true;
5128 : }
5129 :
5130 : void cfs_bandwidth_usage_inc(void) {}
5131 : void cfs_bandwidth_usage_dec(void) {}
5132 : #endif /* CONFIG_JUMP_LABEL */
5133 :
5134 : /*
5135 : * default period for cfs group bandwidth.
5136 : * default: 0.1s, units: nanoseconds
5137 : */
5138 : static inline u64 default_cfs_period(void)
5139 : {
5140 : return 100000000ULL;
5141 : }
5142 :
5143 : static inline u64 sched_cfs_bandwidth_slice(void)
5144 : {
5145 : return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
5146 : }
5147 :
5148 : /*
5149 : * Replenish runtime according to assigned quota. We use sched_clock_cpu
5150 : * directly instead of rq->clock to avoid adding additional synchronization
5151 : * around rq->lock.
5152 : *
5153 : * requires cfs_b->lock
5154 : */
5155 : void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
5156 : {
5157 : s64 runtime;
5158 :
5159 : if (unlikely(cfs_b->quota == RUNTIME_INF))
5160 : return;
5161 :
5162 : cfs_b->runtime += cfs_b->quota;
5163 : runtime = cfs_b->runtime_snap - cfs_b->runtime;
5164 : if (runtime > 0) {
5165 : cfs_b->burst_time += runtime;
5166 : cfs_b->nr_burst++;
5167 : }
5168 :
5169 : cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst);
5170 : cfs_b->runtime_snap = cfs_b->runtime;
5171 : }
5172 :
5173 : static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
5174 : {
5175 : return &tg->cfs_bandwidth;
5176 : }
5177 :
5178 : /* returns 0 on failure to allocate runtime */
5179 : static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
5180 : struct cfs_rq *cfs_rq, u64 target_runtime)
5181 : {
5182 : u64 min_amount, amount = 0;
5183 :
5184 : lockdep_assert_held(&cfs_b->lock);
5185 :
5186 : /* note: this is a positive sum as runtime_remaining <= 0 */
5187 : min_amount = target_runtime - cfs_rq->runtime_remaining;
5188 :
5189 : if (cfs_b->quota == RUNTIME_INF)
5190 : amount = min_amount;
5191 : else {
5192 : start_cfs_bandwidth(cfs_b);
5193 :
5194 : if (cfs_b->runtime > 0) {
5195 : amount = min(cfs_b->runtime, min_amount);
5196 : cfs_b->runtime -= amount;
5197 : cfs_b->idle = 0;
5198 : }
5199 : }
5200 :
5201 : cfs_rq->runtime_remaining += amount;
5202 :
5203 : return cfs_rq->runtime_remaining > 0;
5204 : }
5205 :
5206 : /* returns 0 on failure to allocate runtime */
5207 : static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5208 : {
5209 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5210 : int ret;
5211 :
5212 : raw_spin_lock(&cfs_b->lock);
5213 : ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());
5214 : raw_spin_unlock(&cfs_b->lock);
5215 :
5216 : return ret;
5217 : }
5218 :
5219 : static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
5220 : {
5221 : /* dock delta_exec before expiring quota (as it could span periods) */
5222 : cfs_rq->runtime_remaining -= delta_exec;
5223 :
5224 : if (likely(cfs_rq->runtime_remaining > 0))
5225 : return;
5226 :
5227 : if (cfs_rq->throttled)
5228 : return;
5229 : /*
5230 : * if we're unable to extend our runtime we resched so that the active
5231 : * hierarchy can be throttled
5232 : */
5233 : if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
5234 : resched_curr(rq_of(cfs_rq));
5235 : }
5236 :
5237 : static __always_inline
5238 : void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
5239 : {
5240 : if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
5241 : return;
5242 :
5243 : __account_cfs_rq_runtime(cfs_rq, delta_exec);
5244 : }
5245 :
5246 : static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
5247 : {
5248 : return cfs_bandwidth_used() && cfs_rq->throttled;
5249 : }
5250 :
5251 : /* check whether cfs_rq, or any parent, is throttled */
5252 : static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
5253 : {
5254 : return cfs_bandwidth_used() && cfs_rq->throttle_count;
5255 : }
5256 :
5257 : /*
5258 : * Ensure that neither of the group entities corresponding to src_cpu or
5259 : * dest_cpu are members of a throttled hierarchy when performing group
5260 : * load-balance operations.
5261 : */
5262 : static inline int throttled_lb_pair(struct task_group *tg,
5263 : int src_cpu, int dest_cpu)
5264 : {
5265 : struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
5266 :
5267 : src_cfs_rq = tg->cfs_rq[src_cpu];
5268 : dest_cfs_rq = tg->cfs_rq[dest_cpu];
5269 :
5270 : return throttled_hierarchy(src_cfs_rq) ||
5271 : throttled_hierarchy(dest_cfs_rq);
5272 : }
5273 :
5274 : static int tg_unthrottle_up(struct task_group *tg, void *data)
5275 : {
5276 : struct rq *rq = data;
5277 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5278 :
5279 : cfs_rq->throttle_count--;
5280 : if (!cfs_rq->throttle_count) {
5281 : cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) -
5282 : cfs_rq->throttled_clock_pelt;
5283 :
5284 : /* Add cfs_rq with load or one or more already running entities to the list */
5285 : if (!cfs_rq_is_decayed(cfs_rq))
5286 : list_add_leaf_cfs_rq(cfs_rq);
5287 : }
5288 :
5289 : return 0;
5290 : }
5291 :
5292 : static int tg_throttle_down(struct task_group *tg, void *data)
5293 : {
5294 : struct rq *rq = data;
5295 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5296 :
5297 : /* group is entering throttled state, stop time */
5298 : if (!cfs_rq->throttle_count) {
5299 : cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);
5300 : list_del_leaf_cfs_rq(cfs_rq);
5301 : }
5302 : cfs_rq->throttle_count++;
5303 :
5304 : return 0;
5305 : }
5306 :
5307 : static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
5308 : {
5309 : struct rq *rq = rq_of(cfs_rq);
5310 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5311 : struct sched_entity *se;
5312 : long task_delta, idle_task_delta, dequeue = 1;
5313 :
5314 : raw_spin_lock(&cfs_b->lock);
5315 : /* This will start the period timer if necessary */
5316 : if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {
5317 : /*
5318 : * We have raced with bandwidth becoming available, and if we
5319 : * actually throttled the timer might not unthrottle us for an
5320 : * entire period. We additionally needed to make sure that any
5321 : * subsequent check_cfs_rq_runtime calls agree not to throttle
5322 : * us, as we may commit to do cfs put_prev+pick_next, so we ask
5323 : * for 1ns of runtime rather than just check cfs_b.
5324 : */
5325 : dequeue = 0;
5326 : } else {
5327 : list_add_tail_rcu(&cfs_rq->throttled_list,
5328 : &cfs_b->throttled_cfs_rq);
5329 : }
5330 : raw_spin_unlock(&cfs_b->lock);
5331 :
5332 : if (!dequeue)
5333 : return false; /* Throttle no longer required. */
5334 :
5335 : se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
5336 :
5337 : /* freeze hierarchy runnable averages while throttled */
5338 : rcu_read_lock();
5339 : walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq);
5340 : rcu_read_unlock();
5341 :
5342 : task_delta = cfs_rq->h_nr_running;
5343 : idle_task_delta = cfs_rq->idle_h_nr_running;
5344 : for_each_sched_entity(se) {
5345 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5346 : /* throttled entity or throttle-on-deactivate */
5347 : if (!se->on_rq)
5348 : goto done;
5349 :
5350 : dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
5351 :
5352 : if (cfs_rq_is_idle(group_cfs_rq(se)))
5353 : idle_task_delta = cfs_rq->h_nr_running;
5354 :
5355 : qcfs_rq->h_nr_running -= task_delta;
5356 : qcfs_rq->idle_h_nr_running -= idle_task_delta;
5357 :
5358 : if (qcfs_rq->load.weight) {
5359 : /* Avoid re-evaluating load for this entity: */
5360 : se = parent_entity(se);
5361 : break;
5362 : }
5363 : }
5364 :
5365 : for_each_sched_entity(se) {
5366 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5367 : /* throttled entity or throttle-on-deactivate */
5368 : if (!se->on_rq)
5369 : goto done;
5370 :
5371 : update_load_avg(qcfs_rq, se, 0);
5372 : se_update_runnable(se);
5373 :
5374 : if (cfs_rq_is_idle(group_cfs_rq(se)))
5375 : idle_task_delta = cfs_rq->h_nr_running;
5376 :
5377 : qcfs_rq->h_nr_running -= task_delta;
5378 : qcfs_rq->idle_h_nr_running -= idle_task_delta;
5379 : }
5380 :
5381 : /* At this point se is NULL and we are at root level*/
5382 : sub_nr_running(rq, task_delta);
5383 :
5384 : done:
5385 : /*
5386 : * Note: distribution will already see us throttled via the
5387 : * throttled-list. rq->lock protects completion.
5388 : */
5389 : cfs_rq->throttled = 1;
5390 : cfs_rq->throttled_clock = rq_clock(rq);
5391 : return true;
5392 : }
5393 :
5394 : void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
5395 : {
5396 : struct rq *rq = rq_of(cfs_rq);
5397 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5398 : struct sched_entity *se;
5399 : long task_delta, idle_task_delta;
5400 :
5401 : se = cfs_rq->tg->se[cpu_of(rq)];
5402 :
5403 : cfs_rq->throttled = 0;
5404 :
5405 : update_rq_clock(rq);
5406 :
5407 : raw_spin_lock(&cfs_b->lock);
5408 : cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
5409 : list_del_rcu(&cfs_rq->throttled_list);
5410 : raw_spin_unlock(&cfs_b->lock);
5411 :
5412 : /* update hierarchical throttle state */
5413 : walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq);
5414 :
5415 : if (!cfs_rq->load.weight) {
5416 : if (!cfs_rq->on_list)
5417 : return;
5418 : /*
5419 : * Nothing to run but something to decay (on_list)?
5420 : * Complete the branch.
5421 : */
5422 : for_each_sched_entity(se) {
5423 : if (list_add_leaf_cfs_rq(cfs_rq_of(se)))
5424 : break;
5425 : }
5426 : goto unthrottle_throttle;
5427 : }
5428 :
5429 : task_delta = cfs_rq->h_nr_running;
5430 : idle_task_delta = cfs_rq->idle_h_nr_running;
5431 : for_each_sched_entity(se) {
5432 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5433 :
5434 : if (se->on_rq)
5435 : break;
5436 : enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP);
5437 :
5438 : if (cfs_rq_is_idle(group_cfs_rq(se)))
5439 : idle_task_delta = cfs_rq->h_nr_running;
5440 :
5441 : qcfs_rq->h_nr_running += task_delta;
5442 : qcfs_rq->idle_h_nr_running += idle_task_delta;
5443 :
5444 : /* end evaluation on encountering a throttled cfs_rq */
5445 : if (cfs_rq_throttled(qcfs_rq))
5446 : goto unthrottle_throttle;
5447 : }
5448 :
5449 : for_each_sched_entity(se) {
5450 : struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5451 :
5452 : update_load_avg(qcfs_rq, se, UPDATE_TG);
5453 : se_update_runnable(se);
5454 :
5455 : if (cfs_rq_is_idle(group_cfs_rq(se)))
5456 : idle_task_delta = cfs_rq->h_nr_running;
5457 :
5458 : qcfs_rq->h_nr_running += task_delta;
5459 : qcfs_rq->idle_h_nr_running += idle_task_delta;
5460 :
5461 : /* end evaluation on encountering a throttled cfs_rq */
5462 : if (cfs_rq_throttled(qcfs_rq))
5463 : goto unthrottle_throttle;
5464 : }
5465 :
5466 : /* At this point se is NULL and we are at root level*/
5467 : add_nr_running(rq, task_delta);
5468 :
5469 : unthrottle_throttle:
5470 : assert_list_leaf_cfs_rq(rq);
5471 :
5472 : /* Determine whether we need to wake up potentially idle CPU: */
5473 : if (rq->curr == rq->idle && rq->cfs.nr_running)
5474 : resched_curr(rq);
5475 : }
5476 :
5477 : #ifdef CONFIG_SMP
5478 : static void __cfsb_csd_unthrottle(void *arg)
5479 : {
5480 : struct cfs_rq *cursor, *tmp;
5481 : struct rq *rq = arg;
5482 : struct rq_flags rf;
5483 :
5484 : rq_lock(rq, &rf);
5485 :
5486 : /*
5487 : * Since we hold rq lock we're safe from concurrent manipulation of
5488 : * the CSD list. However, this RCU critical section annotates the
5489 : * fact that we pair with sched_free_group_rcu(), so that we cannot
5490 : * race with group being freed in the window between removing it
5491 : * from the list and advancing to the next entry in the list.
5492 : */
5493 : rcu_read_lock();
5494 :
5495 : list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list,
5496 : throttled_csd_list) {
5497 : list_del_init(&cursor->throttled_csd_list);
5498 :
5499 : if (cfs_rq_throttled(cursor))
5500 : unthrottle_cfs_rq(cursor);
5501 : }
5502 :
5503 : rcu_read_unlock();
5504 :
5505 : rq_unlock(rq, &rf);
5506 : }
5507 :
5508 : static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
5509 : {
5510 : struct rq *rq = rq_of(cfs_rq);
5511 : bool first;
5512 :
5513 : if (rq == this_rq()) {
5514 : unthrottle_cfs_rq(cfs_rq);
5515 : return;
5516 : }
5517 :
5518 : /* Already enqueued */
5519 : if (SCHED_WARN_ON(!list_empty(&cfs_rq->throttled_csd_list)))
5520 : return;
5521 :
5522 : first = list_empty(&rq->cfsb_csd_list);
5523 : list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list);
5524 : if (first)
5525 : smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd);
5526 : }
5527 : #else
5528 : static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
5529 : {
5530 : unthrottle_cfs_rq(cfs_rq);
5531 : }
5532 : #endif
5533 :
5534 : static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
5535 : {
5536 : lockdep_assert_rq_held(rq_of(cfs_rq));
5537 :
5538 : if (SCHED_WARN_ON(!cfs_rq_throttled(cfs_rq) ||
5539 : cfs_rq->runtime_remaining <= 0))
5540 : return;
5541 :
5542 : __unthrottle_cfs_rq_async(cfs_rq);
5543 : }
5544 :
5545 : static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
5546 : {
5547 : struct cfs_rq *local_unthrottle = NULL;
5548 : int this_cpu = smp_processor_id();
5549 : u64 runtime, remaining = 1;
5550 : bool throttled = false;
5551 : struct cfs_rq *cfs_rq;
5552 : struct rq_flags rf;
5553 : struct rq *rq;
5554 :
5555 : rcu_read_lock();
5556 : list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
5557 : throttled_list) {
5558 : rq = rq_of(cfs_rq);
5559 :
5560 : if (!remaining) {
5561 : throttled = true;
5562 : break;
5563 : }
5564 :
5565 : rq_lock_irqsave(rq, &rf);
5566 : if (!cfs_rq_throttled(cfs_rq))
5567 : goto next;
5568 :
5569 : #ifdef CONFIG_SMP
5570 : /* Already queued for async unthrottle */
5571 : if (!list_empty(&cfs_rq->throttled_csd_list))
5572 : goto next;
5573 : #endif
5574 :
5575 : /* By the above checks, this should never be true */
5576 : SCHED_WARN_ON(cfs_rq->runtime_remaining > 0);
5577 :
5578 : raw_spin_lock(&cfs_b->lock);
5579 : runtime = -cfs_rq->runtime_remaining + 1;
5580 : if (runtime > cfs_b->runtime)
5581 : runtime = cfs_b->runtime;
5582 : cfs_b->runtime -= runtime;
5583 : remaining = cfs_b->runtime;
5584 : raw_spin_unlock(&cfs_b->lock);
5585 :
5586 : cfs_rq->runtime_remaining += runtime;
5587 :
5588 : /* we check whether we're throttled above */
5589 : if (cfs_rq->runtime_remaining > 0) {
5590 : if (cpu_of(rq) != this_cpu ||
5591 : SCHED_WARN_ON(local_unthrottle))
5592 : unthrottle_cfs_rq_async(cfs_rq);
5593 : else
5594 : local_unthrottle = cfs_rq;
5595 : } else {
5596 : throttled = true;
5597 : }
5598 :
5599 : next:
5600 : rq_unlock_irqrestore(rq, &rf);
5601 : }
5602 : rcu_read_unlock();
5603 :
5604 : if (local_unthrottle) {
5605 : rq = cpu_rq(this_cpu);
5606 : rq_lock_irqsave(rq, &rf);
5607 : if (cfs_rq_throttled(local_unthrottle))
5608 : unthrottle_cfs_rq(local_unthrottle);
5609 : rq_unlock_irqrestore(rq, &rf);
5610 : }
5611 :
5612 : return throttled;
5613 : }
5614 :
5615 : /*
5616 : * Responsible for refilling a task_group's bandwidth and unthrottling its
5617 : * cfs_rqs as appropriate. If there has been no activity within the last
5618 : * period the timer is deactivated until scheduling resumes; cfs_b->idle is
5619 : * used to track this state.
5620 : */
5621 : static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
5622 : {
5623 : int throttled;
5624 :
5625 : /* no need to continue the timer with no bandwidth constraint */
5626 : if (cfs_b->quota == RUNTIME_INF)
5627 : goto out_deactivate;
5628 :
5629 : throttled = !list_empty(&cfs_b->throttled_cfs_rq);
5630 : cfs_b->nr_periods += overrun;
5631 :
5632 : /* Refill extra burst quota even if cfs_b->idle */
5633 : __refill_cfs_bandwidth_runtime(cfs_b);
5634 :
5635 : /*
5636 : * idle depends on !throttled (for the case of a large deficit), and if
5637 : * we're going inactive then everything else can be deferred
5638 : */
5639 : if (cfs_b->idle && !throttled)
5640 : goto out_deactivate;
5641 :
5642 : if (!throttled) {
5643 : /* mark as potentially idle for the upcoming period */
5644 : cfs_b->idle = 1;
5645 : return 0;
5646 : }
5647 :
5648 : /* account preceding periods in which throttling occurred */
5649 : cfs_b->nr_throttled += overrun;
5650 :
5651 : /*
5652 : * This check is repeated as we release cfs_b->lock while we unthrottle.
5653 : */
5654 : while (throttled && cfs_b->runtime > 0) {
5655 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5656 : /* we can't nest cfs_b->lock while distributing bandwidth */
5657 : throttled = distribute_cfs_runtime(cfs_b);
5658 : raw_spin_lock_irqsave(&cfs_b->lock, flags);
5659 : }
5660 :
5661 : /*
5662 : * While we are ensured activity in the period following an
5663 : * unthrottle, this also covers the case in which the new bandwidth is
5664 : * insufficient to cover the existing bandwidth deficit. (Forcing the
5665 : * timer to remain active while there are any throttled entities.)
5666 : */
5667 : cfs_b->idle = 0;
5668 :
5669 : return 0;
5670 :
5671 : out_deactivate:
5672 : return 1;
5673 : }
5674 :
5675 : /* a cfs_rq won't donate quota below this amount */
5676 : static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
5677 : /* minimum remaining period time to redistribute slack quota */
5678 : static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
5679 : /* how long we wait to gather additional slack before distributing */
5680 : static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
5681 :
5682 : /*
5683 : * Are we near the end of the current quota period?
5684 : *
5685 : * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
5686 : * hrtimer base being cleared by hrtimer_start. In the case of
5687 : * migrate_hrtimers, base is never cleared, so we are fine.
5688 : */
5689 : static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
5690 : {
5691 : struct hrtimer *refresh_timer = &cfs_b->period_timer;
5692 : s64 remaining;
5693 :
5694 : /* if the call-back is running a quota refresh is already occurring */
5695 : if (hrtimer_callback_running(refresh_timer))
5696 : return 1;
5697 :
5698 : /* is a quota refresh about to occur? */
5699 : remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer));
5700 : if (remaining < (s64)min_expire)
5701 : return 1;
5702 :
5703 : return 0;
5704 : }
5705 :
5706 : static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
5707 : {
5708 : u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
5709 :
5710 : /* if there's a quota refresh soon don't bother with slack */
5711 : if (runtime_refresh_within(cfs_b, min_left))
5712 : return;
5713 :
5714 : /* don't push forwards an existing deferred unthrottle */
5715 : if (cfs_b->slack_started)
5716 : return;
5717 : cfs_b->slack_started = true;
5718 :
5719 : hrtimer_start(&cfs_b->slack_timer,
5720 : ns_to_ktime(cfs_bandwidth_slack_period),
5721 : HRTIMER_MODE_REL);
5722 : }
5723 :
5724 : /* we know any runtime found here is valid as update_curr() precedes return */
5725 : static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5726 : {
5727 : struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5728 : s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
5729 :
5730 : if (slack_runtime <= 0)
5731 : return;
5732 :
5733 : raw_spin_lock(&cfs_b->lock);
5734 : if (cfs_b->quota != RUNTIME_INF) {
5735 : cfs_b->runtime += slack_runtime;
5736 :
5737 : /* we are under rq->lock, defer unthrottling using a timer */
5738 : if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
5739 : !list_empty(&cfs_b->throttled_cfs_rq))
5740 : start_cfs_slack_bandwidth(cfs_b);
5741 : }
5742 : raw_spin_unlock(&cfs_b->lock);
5743 :
5744 : /* even if it's not valid for return we don't want to try again */
5745 : cfs_rq->runtime_remaining -= slack_runtime;
5746 : }
5747 :
5748 : static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5749 : {
5750 : if (!cfs_bandwidth_used())
5751 : return;
5752 :
5753 : if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
5754 : return;
5755 :
5756 : __return_cfs_rq_runtime(cfs_rq);
5757 : }
5758 :
5759 : /*
5760 : * This is done with a timer (instead of inline with bandwidth return) since
5761 : * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
5762 : */
5763 : static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
5764 : {
5765 : u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
5766 : unsigned long flags;
5767 :
5768 : /* confirm we're still not at a refresh boundary */
5769 : raw_spin_lock_irqsave(&cfs_b->lock, flags);
5770 : cfs_b->slack_started = false;
5771 :
5772 : if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {
5773 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5774 : return;
5775 : }
5776 :
5777 : if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
5778 : runtime = cfs_b->runtime;
5779 :
5780 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5781 :
5782 : if (!runtime)
5783 : return;
5784 :
5785 : distribute_cfs_runtime(cfs_b);
5786 : }
5787 :
5788 : /*
5789 : * When a group wakes up we want to make sure that its quota is not already
5790 : * expired/exceeded, otherwise it may be allowed to steal additional ticks of
5791 : * runtime as update_curr() throttling can not trigger until it's on-rq.
5792 : */
5793 : static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
5794 : {
5795 : if (!cfs_bandwidth_used())
5796 : return;
5797 :
5798 : /* an active group must be handled by the update_curr()->put() path */
5799 : if (!cfs_rq->runtime_enabled || cfs_rq->curr)
5800 : return;
5801 :
5802 : /* ensure the group is not already throttled */
5803 : if (cfs_rq_throttled(cfs_rq))
5804 : return;
5805 :
5806 : /* update runtime allocation */
5807 : account_cfs_rq_runtime(cfs_rq, 0);
5808 : if (cfs_rq->runtime_remaining <= 0)
5809 : throttle_cfs_rq(cfs_rq);
5810 : }
5811 :
5812 : static void sync_throttle(struct task_group *tg, int cpu)
5813 : {
5814 : struct cfs_rq *pcfs_rq, *cfs_rq;
5815 :
5816 : if (!cfs_bandwidth_used())
5817 : return;
5818 :
5819 : if (!tg->parent)
5820 : return;
5821 :
5822 : cfs_rq = tg->cfs_rq[cpu];
5823 : pcfs_rq = tg->parent->cfs_rq[cpu];
5824 :
5825 : cfs_rq->throttle_count = pcfs_rq->throttle_count;
5826 : cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
5827 : }
5828 :
5829 : /* conditionally throttle active cfs_rq's from put_prev_entity() */
5830 : static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5831 : {
5832 : if (!cfs_bandwidth_used())
5833 : return false;
5834 :
5835 : if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
5836 : return false;
5837 :
5838 : /*
5839 : * it's possible for a throttled entity to be forced into a running
5840 : * state (e.g. set_curr_task), in this case we're finished.
5841 : */
5842 : if (cfs_rq_throttled(cfs_rq))
5843 : return true;
5844 :
5845 : return throttle_cfs_rq(cfs_rq);
5846 : }
5847 :
5848 : static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
5849 : {
5850 : struct cfs_bandwidth *cfs_b =
5851 : container_of(timer, struct cfs_bandwidth, slack_timer);
5852 :
5853 : do_sched_cfs_slack_timer(cfs_b);
5854 :
5855 : return HRTIMER_NORESTART;
5856 : }
5857 :
5858 : extern const u64 max_cfs_quota_period;
5859 :
5860 : static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
5861 : {
5862 : struct cfs_bandwidth *cfs_b =
5863 : container_of(timer, struct cfs_bandwidth, period_timer);
5864 : unsigned long flags;
5865 : int overrun;
5866 : int idle = 0;
5867 : int count = 0;
5868 :
5869 : raw_spin_lock_irqsave(&cfs_b->lock, flags);
5870 : for (;;) {
5871 : overrun = hrtimer_forward_now(timer, cfs_b->period);
5872 : if (!overrun)
5873 : break;
5874 :
5875 : idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);
5876 :
5877 : if (++count > 3) {
5878 : u64 new, old = ktime_to_ns(cfs_b->period);
5879 :
5880 : /*
5881 : * Grow period by a factor of 2 to avoid losing precision.
5882 : * Precision loss in the quota/period ratio can cause __cfs_schedulable
5883 : * to fail.
5884 : */
5885 : new = old * 2;
5886 : if (new < max_cfs_quota_period) {
5887 : cfs_b->period = ns_to_ktime(new);
5888 : cfs_b->quota *= 2;
5889 : cfs_b->burst *= 2;
5890 :
5891 : pr_warn_ratelimited(
5892 : "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n",
5893 : smp_processor_id(),
5894 : div_u64(new, NSEC_PER_USEC),
5895 : div_u64(cfs_b->quota, NSEC_PER_USEC));
5896 : } else {
5897 : pr_warn_ratelimited(
5898 : "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n",
5899 : smp_processor_id(),
5900 : div_u64(old, NSEC_PER_USEC),
5901 : div_u64(cfs_b->quota, NSEC_PER_USEC));
5902 : }
5903 :
5904 : /* reset count so we don't come right back in here */
5905 : count = 0;
5906 : }
5907 : }
5908 : if (idle)
5909 : cfs_b->period_active = 0;
5910 : raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
5911 :
5912 : return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
5913 : }
5914 :
5915 : void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5916 : {
5917 : raw_spin_lock_init(&cfs_b->lock);
5918 : cfs_b->runtime = 0;
5919 : cfs_b->quota = RUNTIME_INF;
5920 : cfs_b->period = ns_to_ktime(default_cfs_period());
5921 : cfs_b->burst = 0;
5922 :
5923 : INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
5924 : hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
5925 : cfs_b->period_timer.function = sched_cfs_period_timer;
5926 : hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
5927 : cfs_b->slack_timer.function = sched_cfs_slack_timer;
5928 : cfs_b->slack_started = false;
5929 : }
5930 :
5931 : static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5932 : {
5933 : cfs_rq->runtime_enabled = 0;
5934 : INIT_LIST_HEAD(&cfs_rq->throttled_list);
5935 : #ifdef CONFIG_SMP
5936 : INIT_LIST_HEAD(&cfs_rq->throttled_csd_list);
5937 : #endif
5938 : }
5939 :
5940 : void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5941 : {
5942 : lockdep_assert_held(&cfs_b->lock);
5943 :
5944 : if (cfs_b->period_active)
5945 : return;
5946 :
5947 : cfs_b->period_active = 1;
5948 : hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
5949 : hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);
5950 : }
5951 :
5952 : static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
5953 : {
5954 : int __maybe_unused i;
5955 :
5956 : /* init_cfs_bandwidth() was not called */
5957 : if (!cfs_b->throttled_cfs_rq.next)
5958 : return;
5959 :
5960 : hrtimer_cancel(&cfs_b->period_timer);
5961 : hrtimer_cancel(&cfs_b->slack_timer);
5962 :
5963 : /*
5964 : * It is possible that we still have some cfs_rq's pending on a CSD
5965 : * list, though this race is very rare. In order for this to occur, we
5966 : * must have raced with the last task leaving the group while there
5967 : * exist throttled cfs_rq(s), and the period_timer must have queued the
5968 : * CSD item but the remote cpu has not yet processed it. To handle this,
5969 : * we can simply flush all pending CSD work inline here. We're
5970 : * guaranteed at this point that no additional cfs_rq of this group can
5971 : * join a CSD list.
5972 : */
5973 : #ifdef CONFIG_SMP
5974 : for_each_possible_cpu(i) {
5975 : struct rq *rq = cpu_rq(i);
5976 : unsigned long flags;
5977 :
5978 : if (list_empty(&rq->cfsb_csd_list))
5979 : continue;
5980 :
5981 : local_irq_save(flags);
5982 : __cfsb_csd_unthrottle(rq);
5983 : local_irq_restore(flags);
5984 : }
5985 : #endif
5986 : }
5987 :
5988 : /*
5989 : * Both these CPU hotplug callbacks race against unregister_fair_sched_group()
5990 : *
5991 : * The race is harmless, since modifying bandwidth settings of unhooked group
5992 : * bits doesn't do much.
5993 : */
5994 :
5995 : /* cpu online callback */
5996 : static void __maybe_unused update_runtime_enabled(struct rq *rq)
5997 : {
5998 : struct task_group *tg;
5999 :
6000 : lockdep_assert_rq_held(rq);
6001 :
6002 : rcu_read_lock();
6003 : list_for_each_entry_rcu(tg, &task_groups, list) {
6004 : struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
6005 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
6006 :
6007 : raw_spin_lock(&cfs_b->lock);
6008 : cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
6009 : raw_spin_unlock(&cfs_b->lock);
6010 : }
6011 : rcu_read_unlock();
6012 : }
6013 :
6014 : /* cpu offline callback */
6015 : static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
6016 : {
6017 : struct task_group *tg;
6018 :
6019 : lockdep_assert_rq_held(rq);
6020 :
6021 : rcu_read_lock();
6022 : list_for_each_entry_rcu(tg, &task_groups, list) {
6023 : struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
6024 :
6025 : if (!cfs_rq->runtime_enabled)
6026 : continue;
6027 :
6028 : /*
6029 : * clock_task is not advancing so we just need to make sure
6030 : * there's some valid quota amount
6031 : */
6032 : cfs_rq->runtime_remaining = 1;
6033 : /*
6034 : * Offline rq is schedulable till CPU is completely disabled
6035 : * in take_cpu_down(), so we prevent new cfs throttling here.
6036 : */
6037 : cfs_rq->runtime_enabled = 0;
6038 :
6039 : if (cfs_rq_throttled(cfs_rq))
6040 : unthrottle_cfs_rq(cfs_rq);
6041 : }
6042 : rcu_read_unlock();
6043 : }
6044 :
6045 : #else /* CONFIG_CFS_BANDWIDTH */
6046 :
6047 : static inline bool cfs_bandwidth_used(void)
6048 : {
6049 : return false;
6050 : }
6051 :
6052 : static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
6053 : static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
6054 : static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
6055 : static inline void sync_throttle(struct task_group *tg, int cpu) {}
6056 : static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
6057 :
6058 : static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
6059 : {
6060 : return 0;
6061 : }
6062 :
6063 : static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
6064 : {
6065 : return 0;
6066 : }
6067 :
6068 : static inline int throttled_lb_pair(struct task_group *tg,
6069 : int src_cpu, int dest_cpu)
6070 : {
6071 : return 0;
6072 : }
6073 :
6074 0 : void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
6075 :
6076 : #ifdef CONFIG_FAIR_GROUP_SCHED
6077 : static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
6078 : #endif
6079 :
6080 : static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
6081 : {
6082 : return NULL;
6083 : }
6084 : static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
6085 : static inline void update_runtime_enabled(struct rq *rq) {}
6086 : static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
6087 :
6088 : #endif /* CONFIG_CFS_BANDWIDTH */
6089 :
6090 : /**************************************************
6091 : * CFS operations on tasks:
6092 : */
6093 :
6094 : #ifdef CONFIG_SCHED_HRTICK
6095 : static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
6096 : {
6097 : struct sched_entity *se = &p->se;
6098 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
6099 :
6100 : SCHED_WARN_ON(task_rq(p) != rq);
6101 :
6102 : if (rq->cfs.h_nr_running > 1) {
6103 : u64 slice = sched_slice(cfs_rq, se);
6104 : u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
6105 : s64 delta = slice - ran;
6106 :
6107 : if (delta < 0) {
6108 : if (task_current(rq, p))
6109 : resched_curr(rq);
6110 : return;
6111 : }
6112 : hrtick_start(rq, delta);
6113 : }
6114 : }
6115 :
6116 : /*
6117 : * called from enqueue/dequeue and updates the hrtick when the
6118 : * current task is from our class and nr_running is low enough
6119 : * to matter.
6120 : */
6121 : static void hrtick_update(struct rq *rq)
6122 : {
6123 : struct task_struct *curr = rq->curr;
6124 :
6125 : if (!hrtick_enabled_fair(rq) || curr->sched_class != &fair_sched_class)
6126 : return;
6127 :
6128 : if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency)
6129 : hrtick_start_fair(rq, curr);
6130 : }
6131 : #else /* !CONFIG_SCHED_HRTICK */
6132 : static inline void
6133 : hrtick_start_fair(struct rq *rq, struct task_struct *p)
6134 : {
6135 : }
6136 :
6137 : static inline void hrtick_update(struct rq *rq)
6138 : {
6139 : }
6140 : #endif
6141 :
6142 : #ifdef CONFIG_SMP
6143 : static inline bool cpu_overutilized(int cpu)
6144 : {
6145 : unsigned long rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN);
6146 : unsigned long rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
6147 :
6148 : /* Return true only if the utilization doesn't fit CPU's capacity */
6149 : return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu);
6150 : }
6151 :
6152 : static inline void update_overutilized_status(struct rq *rq)
6153 : {
6154 : if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) {
6155 : WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED);
6156 : trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED);
6157 : }
6158 : }
6159 : #else
6160 : static inline void update_overutilized_status(struct rq *rq) { }
6161 : #endif
6162 :
6163 : /* Runqueue only has SCHED_IDLE tasks enqueued */
6164 : static int sched_idle_rq(struct rq *rq)
6165 : {
6166 4260 : return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running &&
6167 : rq->nr_running);
6168 : }
6169 :
6170 : /*
6171 : * Returns true if cfs_rq only has SCHED_IDLE entities enqueued. Note the use
6172 : * of idle_nr_running, which does not consider idle descendants of normal
6173 : * entities.
6174 : */
6175 : static bool sched_idle_cfs_rq(struct cfs_rq *cfs_rq)
6176 : {
6177 : return cfs_rq->nr_running &&
6178 : cfs_rq->nr_running == cfs_rq->idle_nr_running;
6179 : }
6180 :
6181 : #ifdef CONFIG_SMP
6182 : static int sched_idle_cpu(int cpu)
6183 : {
6184 : return sched_idle_rq(cpu_rq(cpu));
6185 : }
6186 : #endif
6187 :
6188 : /*
6189 : * The enqueue_task method is called before nr_running is
6190 : * increased. Here we update the fair scheduling stats and
6191 : * then put the task into the rbtree:
6192 : */
6193 : static void
6194 2132 : enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
6195 : {
6196 : struct cfs_rq *cfs_rq;
6197 2132 : struct sched_entity *se = &p->se;
6198 4264 : int idle_h_nr_running = task_has_idle_policy(p);
6199 2132 : int task_new = !(flags & ENQUEUE_WAKEUP);
6200 :
6201 : /*
6202 : * The code below (indirectly) updates schedutil which looks at
6203 : * the cfs_rq utilization to select a frequency.
6204 : * Let's add the task's estimated utilization to the cfs_rq's
6205 : * estimated utilization, before we update schedutil.
6206 : */
6207 2132 : util_est_enqueue(&rq->cfs, p);
6208 :
6209 : /*
6210 : * If in_iowait is set, the code below may not trigger any cpufreq
6211 : * utilization updates, so do it here explicitly with the IOWAIT flag
6212 : * passed.
6213 : */
6214 : if (p->in_iowait)
6215 : cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
6216 :
6217 4264 : for_each_sched_entity(se) {
6218 2132 : if (se->on_rq)
6219 : break;
6220 4264 : cfs_rq = cfs_rq_of(se);
6221 2132 : enqueue_entity(cfs_rq, se, flags);
6222 :
6223 2132 : cfs_rq->h_nr_running++;
6224 2132 : cfs_rq->idle_h_nr_running += idle_h_nr_running;
6225 :
6226 : if (cfs_rq_is_idle(cfs_rq))
6227 : idle_h_nr_running = 1;
6228 :
6229 : /* end evaluation on encountering a throttled cfs_rq */
6230 : if (cfs_rq_throttled(cfs_rq))
6231 : goto enqueue_throttle;
6232 :
6233 : flags = ENQUEUE_WAKEUP;
6234 : }
6235 :
6236 2132 : for_each_sched_entity(se) {
6237 0 : cfs_rq = cfs_rq_of(se);
6238 :
6239 0 : update_load_avg(cfs_rq, se, UPDATE_TG);
6240 0 : se_update_runnable(se);
6241 0 : update_cfs_group(se);
6242 :
6243 0 : cfs_rq->h_nr_running++;
6244 0 : cfs_rq->idle_h_nr_running += idle_h_nr_running;
6245 :
6246 : if (cfs_rq_is_idle(cfs_rq))
6247 : idle_h_nr_running = 1;
6248 :
6249 : /* end evaluation on encountering a throttled cfs_rq */
6250 : if (cfs_rq_throttled(cfs_rq))
6251 : goto enqueue_throttle;
6252 : }
6253 :
6254 : /* At this point se is NULL and we are at root level*/
6255 4264 : add_nr_running(rq, 1);
6256 :
6257 : /*
6258 : * Since new tasks are assigned an initial util_avg equal to
6259 : * half of the spare capacity of their CPU, tiny tasks have the
6260 : * ability to cross the overutilized threshold, which will
6261 : * result in the load balancer ruining all the task placement
6262 : * done by EAS. As a way to mitigate that effect, do not account
6263 : * for the first enqueue operation of new tasks during the
6264 : * overutilized flag detection.
6265 : *
6266 : * A better way of solving this problem would be to wait for
6267 : * the PELT signals of tasks to converge before taking them
6268 : * into account, but that is not straightforward to implement,
6269 : * and the following generally works well enough in practice.
6270 : */
6271 : if (!task_new)
6272 : update_overutilized_status(rq);
6273 :
6274 : enqueue_throttle:
6275 2132 : assert_list_leaf_cfs_rq(rq);
6276 :
6277 2132 : hrtick_update(rq);
6278 2132 : }
6279 :
6280 : static void set_next_buddy(struct sched_entity *se);
6281 :
6282 : /*
6283 : * The dequeue_task method is called before nr_running is
6284 : * decreased. We remove the task from the rbtree and
6285 : * update the fair scheduling stats:
6286 : */
6287 2130 : static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
6288 : {
6289 : struct cfs_rq *cfs_rq;
6290 2130 : struct sched_entity *se = &p->se;
6291 2130 : int task_sleep = flags & DEQUEUE_SLEEP;
6292 4260 : int idle_h_nr_running = task_has_idle_policy(p);
6293 4260 : bool was_sched_idle = sched_idle_rq(rq);
6294 :
6295 2130 : util_est_dequeue(&rq->cfs, p);
6296 :
6297 1 : for_each_sched_entity(se) {
6298 4260 : cfs_rq = cfs_rq_of(se);
6299 2130 : dequeue_entity(cfs_rq, se, flags);
6300 :
6301 2130 : cfs_rq->h_nr_running--;
6302 2130 : cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6303 :
6304 : if (cfs_rq_is_idle(cfs_rq))
6305 : idle_h_nr_running = 1;
6306 :
6307 : /* end evaluation on encountering a throttled cfs_rq */
6308 : if (cfs_rq_throttled(cfs_rq))
6309 : goto dequeue_throttle;
6310 :
6311 : /* Don't dequeue parent if it has other entities besides us */
6312 2130 : if (cfs_rq->load.weight) {
6313 : /* Avoid re-evaluating load for this entity: */
6314 : se = parent_entity(se);
6315 : /*
6316 : * Bias pick_next to pick a task from this cfs_rq, as
6317 : * p is sleeping when it is within its sched_slice.
6318 : */
6319 : if (task_sleep && se && !throttled_hierarchy(cfs_rq))
6320 : set_next_buddy(se);
6321 : break;
6322 : }
6323 1 : flags |= DEQUEUE_SLEEP;
6324 : }
6325 :
6326 2130 : for_each_sched_entity(se) {
6327 0 : cfs_rq = cfs_rq_of(se);
6328 :
6329 0 : update_load_avg(cfs_rq, se, UPDATE_TG);
6330 0 : se_update_runnable(se);
6331 0 : update_cfs_group(se);
6332 :
6333 0 : cfs_rq->h_nr_running--;
6334 0 : cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6335 :
6336 : if (cfs_rq_is_idle(cfs_rq))
6337 : idle_h_nr_running = 1;
6338 :
6339 : /* end evaluation on encountering a throttled cfs_rq */
6340 : if (cfs_rq_throttled(cfs_rq))
6341 : goto dequeue_throttle;
6342 :
6343 : }
6344 :
6345 : /* At this point se is NULL and we are at root level*/
6346 4260 : sub_nr_running(rq, 1);
6347 :
6348 : /* balance early to pull high priority tasks */
6349 4260 : if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
6350 0 : rq->next_balance = jiffies;
6351 :
6352 : dequeue_throttle:
6353 2130 : util_est_update(&rq->cfs, p, task_sleep);
6354 2130 : hrtick_update(rq);
6355 2130 : }
6356 :
6357 : #ifdef CONFIG_SMP
6358 :
6359 : /* Working cpumask for: load_balance, load_balance_newidle. */
6360 : static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
6361 : static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask);
6362 :
6363 : #ifdef CONFIG_NO_HZ_COMMON
6364 :
6365 : static struct {
6366 : cpumask_var_t idle_cpus_mask;
6367 : atomic_t nr_cpus;
6368 : int has_blocked; /* Idle CPUS has blocked load */
6369 : int needs_update; /* Newly idle CPUs need their next_balance collated */
6370 : unsigned long next_balance; /* in jiffy units */
6371 : unsigned long next_blocked; /* Next update of blocked load in jiffies */
6372 : } nohz ____cacheline_aligned;
6373 :
6374 : #endif /* CONFIG_NO_HZ_COMMON */
6375 :
6376 : static unsigned long cpu_load(struct rq *rq)
6377 : {
6378 : return cfs_rq_load_avg(&rq->cfs);
6379 : }
6380 :
6381 : /*
6382 : * cpu_load_without - compute CPU load without any contributions from *p
6383 : * @cpu: the CPU which load is requested
6384 : * @p: the task which load should be discounted
6385 : *
6386 : * The load of a CPU is defined by the load of tasks currently enqueued on that
6387 : * CPU as well as tasks which are currently sleeping after an execution on that
6388 : * CPU.
6389 : *
6390 : * This method returns the load of the specified CPU by discounting the load of
6391 : * the specified task, whenever the task is currently contributing to the CPU
6392 : * load.
6393 : */
6394 : static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p)
6395 : {
6396 : struct cfs_rq *cfs_rq;
6397 : unsigned int load;
6398 :
6399 : /* Task has no contribution or is new */
6400 : if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6401 : return cpu_load(rq);
6402 :
6403 : cfs_rq = &rq->cfs;
6404 : load = READ_ONCE(cfs_rq->avg.load_avg);
6405 :
6406 : /* Discount task's util from CPU's util */
6407 : lsub_positive(&load, task_h_load(p));
6408 :
6409 : return load;
6410 : }
6411 :
6412 : static unsigned long cpu_runnable(struct rq *rq)
6413 : {
6414 : return cfs_rq_runnable_avg(&rq->cfs);
6415 : }
6416 :
6417 : static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p)
6418 : {
6419 : struct cfs_rq *cfs_rq;
6420 : unsigned int runnable;
6421 :
6422 : /* Task has no contribution or is new */
6423 : if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6424 : return cpu_runnable(rq);
6425 :
6426 : cfs_rq = &rq->cfs;
6427 : runnable = READ_ONCE(cfs_rq->avg.runnable_avg);
6428 :
6429 : /* Discount task's runnable from CPU's runnable */
6430 : lsub_positive(&runnable, p->se.avg.runnable_avg);
6431 :
6432 : return runnable;
6433 : }
6434 :
6435 : static unsigned long capacity_of(int cpu)
6436 : {
6437 : return cpu_rq(cpu)->cpu_capacity;
6438 : }
6439 :
6440 : static void record_wakee(struct task_struct *p)
6441 : {
6442 : /*
6443 : * Only decay a single time; tasks that have less then 1 wakeup per
6444 : * jiffy will not have built up many flips.
6445 : */
6446 : if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) {
6447 : current->wakee_flips >>= 1;
6448 : current->wakee_flip_decay_ts = jiffies;
6449 : }
6450 :
6451 : if (current->last_wakee != p) {
6452 : current->last_wakee = p;
6453 : current->wakee_flips++;
6454 : }
6455 : }
6456 :
6457 : /*
6458 : * Detect M:N waker/wakee relationships via a switching-frequency heuristic.
6459 : *
6460 : * A waker of many should wake a different task than the one last awakened
6461 : * at a frequency roughly N times higher than one of its wakees.
6462 : *
6463 : * In order to determine whether we should let the load spread vs consolidating
6464 : * to shared cache, we look for a minimum 'flip' frequency of llc_size in one
6465 : * partner, and a factor of lls_size higher frequency in the other.
6466 : *
6467 : * With both conditions met, we can be relatively sure that the relationship is
6468 : * non-monogamous, with partner count exceeding socket size.
6469 : *
6470 : * Waker/wakee being client/server, worker/dispatcher, interrupt source or
6471 : * whatever is irrelevant, spread criteria is apparent partner count exceeds
6472 : * socket size.
6473 : */
6474 : static int wake_wide(struct task_struct *p)
6475 : {
6476 : unsigned int master = current->wakee_flips;
6477 : unsigned int slave = p->wakee_flips;
6478 : int factor = __this_cpu_read(sd_llc_size);
6479 :
6480 : if (master < slave)
6481 : swap(master, slave);
6482 : if (slave < factor || master < slave * factor)
6483 : return 0;
6484 : return 1;
6485 : }
6486 :
6487 : /*
6488 : * The purpose of wake_affine() is to quickly determine on which CPU we can run
6489 : * soonest. For the purpose of speed we only consider the waking and previous
6490 : * CPU.
6491 : *
6492 : * wake_affine_idle() - only considers 'now', it check if the waking CPU is
6493 : * cache-affine and is (or will be) idle.
6494 : *
6495 : * wake_affine_weight() - considers the weight to reflect the average
6496 : * scheduling latency of the CPUs. This seems to work
6497 : * for the overloaded case.
6498 : */
6499 : static int
6500 : wake_affine_idle(int this_cpu, int prev_cpu, int sync)
6501 : {
6502 : /*
6503 : * If this_cpu is idle, it implies the wakeup is from interrupt
6504 : * context. Only allow the move if cache is shared. Otherwise an
6505 : * interrupt intensive workload could force all tasks onto one
6506 : * node depending on the IO topology or IRQ affinity settings.
6507 : *
6508 : * If the prev_cpu is idle and cache affine then avoid a migration.
6509 : * There is no guarantee that the cache hot data from an interrupt
6510 : * is more important than cache hot data on the prev_cpu and from
6511 : * a cpufreq perspective, it's better to have higher utilisation
6512 : * on one CPU.
6513 : */
6514 : if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu))
6515 : return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu;
6516 :
6517 : if (sync && cpu_rq(this_cpu)->nr_running == 1)
6518 : return this_cpu;
6519 :
6520 : if (available_idle_cpu(prev_cpu))
6521 : return prev_cpu;
6522 :
6523 : return nr_cpumask_bits;
6524 : }
6525 :
6526 : static int
6527 : wake_affine_weight(struct sched_domain *sd, struct task_struct *p,
6528 : int this_cpu, int prev_cpu, int sync)
6529 : {
6530 : s64 this_eff_load, prev_eff_load;
6531 : unsigned long task_load;
6532 :
6533 : this_eff_load = cpu_load(cpu_rq(this_cpu));
6534 :
6535 : if (sync) {
6536 : unsigned long current_load = task_h_load(current);
6537 :
6538 : if (current_load > this_eff_load)
6539 : return this_cpu;
6540 :
6541 : this_eff_load -= current_load;
6542 : }
6543 :
6544 : task_load = task_h_load(p);
6545 :
6546 : this_eff_load += task_load;
6547 : if (sched_feat(WA_BIAS))
6548 : this_eff_load *= 100;
6549 : this_eff_load *= capacity_of(prev_cpu);
6550 :
6551 : prev_eff_load = cpu_load(cpu_rq(prev_cpu));
6552 : prev_eff_load -= task_load;
6553 : if (sched_feat(WA_BIAS))
6554 : prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
6555 : prev_eff_load *= capacity_of(this_cpu);
6556 :
6557 : /*
6558 : * If sync, adjust the weight of prev_eff_load such that if
6559 : * prev_eff == this_eff that select_idle_sibling() will consider
6560 : * stacking the wakee on top of the waker if no other CPU is
6561 : * idle.
6562 : */
6563 : if (sync)
6564 : prev_eff_load += 1;
6565 :
6566 : return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits;
6567 : }
6568 :
6569 : static int wake_affine(struct sched_domain *sd, struct task_struct *p,
6570 : int this_cpu, int prev_cpu, int sync)
6571 : {
6572 : int target = nr_cpumask_bits;
6573 :
6574 : if (sched_feat(WA_IDLE))
6575 : target = wake_affine_idle(this_cpu, prev_cpu, sync);
6576 :
6577 : if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits)
6578 : target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
6579 :
6580 : schedstat_inc(p->stats.nr_wakeups_affine_attempts);
6581 : if (target == nr_cpumask_bits)
6582 : return prev_cpu;
6583 :
6584 : schedstat_inc(sd->ttwu_move_affine);
6585 : schedstat_inc(p->stats.nr_wakeups_affine);
6586 : return target;
6587 : }
6588 :
6589 : static struct sched_group *
6590 : find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu);
6591 :
6592 : /*
6593 : * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group.
6594 : */
6595 : static int
6596 : find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
6597 : {
6598 : unsigned long load, min_load = ULONG_MAX;
6599 : unsigned int min_exit_latency = UINT_MAX;
6600 : u64 latest_idle_timestamp = 0;
6601 : int least_loaded_cpu = this_cpu;
6602 : int shallowest_idle_cpu = -1;
6603 : int i;
6604 :
6605 : /* Check if we have any choice: */
6606 : if (group->group_weight == 1)
6607 : return cpumask_first(sched_group_span(group));
6608 :
6609 : /* Traverse only the allowed CPUs */
6610 : for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
6611 : struct rq *rq = cpu_rq(i);
6612 :
6613 : if (!sched_core_cookie_match(rq, p))
6614 : continue;
6615 :
6616 : if (sched_idle_cpu(i))
6617 : return i;
6618 :
6619 : if (available_idle_cpu(i)) {
6620 : struct cpuidle_state *idle = idle_get_state(rq);
6621 : if (idle && idle->exit_latency < min_exit_latency) {
6622 : /*
6623 : * We give priority to a CPU whose idle state
6624 : * has the smallest exit latency irrespective
6625 : * of any idle timestamp.
6626 : */
6627 : min_exit_latency = idle->exit_latency;
6628 : latest_idle_timestamp = rq->idle_stamp;
6629 : shallowest_idle_cpu = i;
6630 : } else if ((!idle || idle->exit_latency == min_exit_latency) &&
6631 : rq->idle_stamp > latest_idle_timestamp) {
6632 : /*
6633 : * If equal or no active idle state, then
6634 : * the most recently idled CPU might have
6635 : * a warmer cache.
6636 : */
6637 : latest_idle_timestamp = rq->idle_stamp;
6638 : shallowest_idle_cpu = i;
6639 : }
6640 : } else if (shallowest_idle_cpu == -1) {
6641 : load = cpu_load(cpu_rq(i));
6642 : if (load < min_load) {
6643 : min_load = load;
6644 : least_loaded_cpu = i;
6645 : }
6646 : }
6647 : }
6648 :
6649 : return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
6650 : }
6651 :
6652 : static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p,
6653 : int cpu, int prev_cpu, int sd_flag)
6654 : {
6655 : int new_cpu = cpu;
6656 :
6657 : if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr))
6658 : return prev_cpu;
6659 :
6660 : /*
6661 : * We need task's util for cpu_util_without, sync it up to
6662 : * prev_cpu's last_update_time.
6663 : */
6664 : if (!(sd_flag & SD_BALANCE_FORK))
6665 : sync_entity_load_avg(&p->se);
6666 :
6667 : while (sd) {
6668 : struct sched_group *group;
6669 : struct sched_domain *tmp;
6670 : int weight;
6671 :
6672 : if (!(sd->flags & sd_flag)) {
6673 : sd = sd->child;
6674 : continue;
6675 : }
6676 :
6677 : group = find_idlest_group(sd, p, cpu);
6678 : if (!group) {
6679 : sd = sd->child;
6680 : continue;
6681 : }
6682 :
6683 : new_cpu = find_idlest_group_cpu(group, p, cpu);
6684 : if (new_cpu == cpu) {
6685 : /* Now try balancing at a lower domain level of 'cpu': */
6686 : sd = sd->child;
6687 : continue;
6688 : }
6689 :
6690 : /* Now try balancing at a lower domain level of 'new_cpu': */
6691 : cpu = new_cpu;
6692 : weight = sd->span_weight;
6693 : sd = NULL;
6694 : for_each_domain(cpu, tmp) {
6695 : if (weight <= tmp->span_weight)
6696 : break;
6697 : if (tmp->flags & sd_flag)
6698 : sd = tmp;
6699 : }
6700 : }
6701 :
6702 : return new_cpu;
6703 : }
6704 :
6705 : static inline int __select_idle_cpu(int cpu, struct task_struct *p)
6706 : {
6707 : if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) &&
6708 : sched_cpu_cookie_match(cpu_rq(cpu), p))
6709 : return cpu;
6710 :
6711 : return -1;
6712 : }
6713 :
6714 : #ifdef CONFIG_SCHED_SMT
6715 : DEFINE_STATIC_KEY_FALSE(sched_smt_present);
6716 : EXPORT_SYMBOL_GPL(sched_smt_present);
6717 :
6718 : static inline void set_idle_cores(int cpu, int val)
6719 : {
6720 : struct sched_domain_shared *sds;
6721 :
6722 : sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6723 : if (sds)
6724 : WRITE_ONCE(sds->has_idle_cores, val);
6725 : }
6726 :
6727 : static inline bool test_idle_cores(int cpu)
6728 : {
6729 : struct sched_domain_shared *sds;
6730 :
6731 : sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
6732 : if (sds)
6733 : return READ_ONCE(sds->has_idle_cores);
6734 :
6735 : return false;
6736 : }
6737 :
6738 : /*
6739 : * Scans the local SMT mask to see if the entire core is idle, and records this
6740 : * information in sd_llc_shared->has_idle_cores.
6741 : *
6742 : * Since SMT siblings share all cache levels, inspecting this limited remote
6743 : * state should be fairly cheap.
6744 : */
6745 : void __update_idle_core(struct rq *rq)
6746 : {
6747 : int core = cpu_of(rq);
6748 : int cpu;
6749 :
6750 : rcu_read_lock();
6751 : if (test_idle_cores(core))
6752 : goto unlock;
6753 :
6754 : for_each_cpu(cpu, cpu_smt_mask(core)) {
6755 : if (cpu == core)
6756 : continue;
6757 :
6758 : if (!available_idle_cpu(cpu))
6759 : goto unlock;
6760 : }
6761 :
6762 : set_idle_cores(core, 1);
6763 : unlock:
6764 : rcu_read_unlock();
6765 : }
6766 :
6767 : /*
6768 : * Scan the entire LLC domain for idle cores; this dynamically switches off if
6769 : * there are no idle cores left in the system; tracked through
6770 : * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
6771 : */
6772 : static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
6773 : {
6774 : bool idle = true;
6775 : int cpu;
6776 :
6777 : for_each_cpu(cpu, cpu_smt_mask(core)) {
6778 : if (!available_idle_cpu(cpu)) {
6779 : idle = false;
6780 : if (*idle_cpu == -1) {
6781 : if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->cpus_ptr)) {
6782 : *idle_cpu = cpu;
6783 : break;
6784 : }
6785 : continue;
6786 : }
6787 : break;
6788 : }
6789 : if (*idle_cpu == -1 && cpumask_test_cpu(cpu, p->cpus_ptr))
6790 : *idle_cpu = cpu;
6791 : }
6792 :
6793 : if (idle)
6794 : return core;
6795 :
6796 : cpumask_andnot(cpus, cpus, cpu_smt_mask(core));
6797 : return -1;
6798 : }
6799 :
6800 : /*
6801 : * Scan the local SMT mask for idle CPUs.
6802 : */
6803 : static int select_idle_smt(struct task_struct *p, int target)
6804 : {
6805 : int cpu;
6806 :
6807 : for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) {
6808 : if (cpu == target)
6809 : continue;
6810 : if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
6811 : return cpu;
6812 : }
6813 :
6814 : return -1;
6815 : }
6816 :
6817 : #else /* CONFIG_SCHED_SMT */
6818 :
6819 : static inline void set_idle_cores(int cpu, int val)
6820 : {
6821 : }
6822 :
6823 : static inline bool test_idle_cores(int cpu)
6824 : {
6825 : return false;
6826 : }
6827 :
6828 : static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
6829 : {
6830 : return __select_idle_cpu(core, p);
6831 : }
6832 :
6833 : static inline int select_idle_smt(struct task_struct *p, int target)
6834 : {
6835 : return -1;
6836 : }
6837 :
6838 : #endif /* CONFIG_SCHED_SMT */
6839 :
6840 : /*
6841 : * Scan the LLC domain for idle CPUs; this is dynamically regulated by
6842 : * comparing the average scan cost (tracked in sd->avg_scan_cost) against the
6843 : * average idle time for this rq (as found in rq->avg_idle).
6844 : */
6845 : static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target)
6846 : {
6847 : struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
6848 : int i, cpu, idle_cpu = -1, nr = INT_MAX;
6849 : struct sched_domain_shared *sd_share;
6850 : struct rq *this_rq = this_rq();
6851 : int this = smp_processor_id();
6852 : struct sched_domain *this_sd = NULL;
6853 : u64 time = 0;
6854 :
6855 : cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
6856 :
6857 : if (sched_feat(SIS_PROP) && !has_idle_core) {
6858 : u64 avg_cost, avg_idle, span_avg;
6859 : unsigned long now = jiffies;
6860 :
6861 : this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc));
6862 : if (!this_sd)
6863 : return -1;
6864 :
6865 : /*
6866 : * If we're busy, the assumption that the last idle period
6867 : * predicts the future is flawed; age away the remaining
6868 : * predicted idle time.
6869 : */
6870 : if (unlikely(this_rq->wake_stamp < now)) {
6871 : while (this_rq->wake_stamp < now && this_rq->wake_avg_idle) {
6872 : this_rq->wake_stamp++;
6873 : this_rq->wake_avg_idle >>= 1;
6874 : }
6875 : }
6876 :
6877 : avg_idle = this_rq->wake_avg_idle;
6878 : avg_cost = this_sd->avg_scan_cost + 1;
6879 :
6880 : span_avg = sd->span_weight * avg_idle;
6881 : if (span_avg > 4*avg_cost)
6882 : nr = div_u64(span_avg, avg_cost);
6883 : else
6884 : nr = 4;
6885 :
6886 : time = cpu_clock(this);
6887 : }
6888 :
6889 : if (sched_feat(SIS_UTIL)) {
6890 : sd_share = rcu_dereference(per_cpu(sd_llc_shared, target));
6891 : if (sd_share) {
6892 : /* because !--nr is the condition to stop scan */
6893 : nr = READ_ONCE(sd_share->nr_idle_scan) + 1;
6894 : /* overloaded LLC is unlikely to have idle cpu/core */
6895 : if (nr == 1)
6896 : return -1;
6897 : }
6898 : }
6899 :
6900 : for_each_cpu_wrap(cpu, cpus, target + 1) {
6901 : if (has_idle_core) {
6902 : i = select_idle_core(p, cpu, cpus, &idle_cpu);
6903 : if ((unsigned int)i < nr_cpumask_bits)
6904 : return i;
6905 :
6906 : } else {
6907 : if (!--nr)
6908 : return -1;
6909 : idle_cpu = __select_idle_cpu(cpu, p);
6910 : if ((unsigned int)idle_cpu < nr_cpumask_bits)
6911 : break;
6912 : }
6913 : }
6914 :
6915 : if (has_idle_core)
6916 : set_idle_cores(target, false);
6917 :
6918 : if (sched_feat(SIS_PROP) && this_sd && !has_idle_core) {
6919 : time = cpu_clock(this) - time;
6920 :
6921 : /*
6922 : * Account for the scan cost of wakeups against the average
6923 : * idle time.
6924 : */
6925 : this_rq->wake_avg_idle -= min(this_rq->wake_avg_idle, time);
6926 :
6927 : update_avg(&this_sd->avg_scan_cost, time);
6928 : }
6929 :
6930 : return idle_cpu;
6931 : }
6932 :
6933 : /*
6934 : * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which
6935 : * the task fits. If no CPU is big enough, but there are idle ones, try to
6936 : * maximize capacity.
6937 : */
6938 : static int
6939 : select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
6940 : {
6941 : unsigned long task_util, util_min, util_max, best_cap = 0;
6942 : int fits, best_fits = 0;
6943 : int cpu, best_cpu = -1;
6944 : struct cpumask *cpus;
6945 :
6946 : cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
6947 : cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
6948 :
6949 : task_util = task_util_est(p);
6950 : util_min = uclamp_eff_value(p, UCLAMP_MIN);
6951 : util_max = uclamp_eff_value(p, UCLAMP_MAX);
6952 :
6953 : for_each_cpu_wrap(cpu, cpus, target + 1) {
6954 : unsigned long cpu_cap = capacity_of(cpu);
6955 :
6956 : if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu))
6957 : continue;
6958 :
6959 : fits = util_fits_cpu(task_util, util_min, util_max, cpu);
6960 :
6961 : /* This CPU fits with all requirements */
6962 : if (fits > 0)
6963 : return cpu;
6964 : /*
6965 : * Only the min performance hint (i.e. uclamp_min) doesn't fit.
6966 : * Look for the CPU with best capacity.
6967 : */
6968 : else if (fits < 0)
6969 : cpu_cap = capacity_orig_of(cpu) - thermal_load_avg(cpu_rq(cpu));
6970 :
6971 : /*
6972 : * First, select CPU which fits better (-1 being better than 0).
6973 : * Then, select the one with best capacity at same level.
6974 : */
6975 : if ((fits < best_fits) ||
6976 : ((fits == best_fits) && (cpu_cap > best_cap))) {
6977 : best_cap = cpu_cap;
6978 : best_cpu = cpu;
6979 : best_fits = fits;
6980 : }
6981 : }
6982 :
6983 : return best_cpu;
6984 : }
6985 :
6986 : static inline bool asym_fits_cpu(unsigned long util,
6987 : unsigned long util_min,
6988 : unsigned long util_max,
6989 : int cpu)
6990 : {
6991 : if (sched_asym_cpucap_active())
6992 : /*
6993 : * Return true only if the cpu fully fits the task requirements
6994 : * which include the utilization and the performance hints.
6995 : */
6996 : return (util_fits_cpu(util, util_min, util_max, cpu) > 0);
6997 :
6998 : return true;
6999 : }
7000 :
7001 : /*
7002 : * Try and locate an idle core/thread in the LLC cache domain.
7003 : */
7004 : static int select_idle_sibling(struct task_struct *p, int prev, int target)
7005 : {
7006 : bool has_idle_core = false;
7007 : struct sched_domain *sd;
7008 : unsigned long task_util, util_min, util_max;
7009 : int i, recent_used_cpu;
7010 :
7011 : /*
7012 : * On asymmetric system, update task utilization because we will check
7013 : * that the task fits with cpu's capacity.
7014 : */
7015 : if (sched_asym_cpucap_active()) {
7016 : sync_entity_load_avg(&p->se);
7017 : task_util = task_util_est(p);
7018 : util_min = uclamp_eff_value(p, UCLAMP_MIN);
7019 : util_max = uclamp_eff_value(p, UCLAMP_MAX);
7020 : }
7021 :
7022 : /*
7023 : * per-cpu select_rq_mask usage
7024 : */
7025 : lockdep_assert_irqs_disabled();
7026 :
7027 : if ((available_idle_cpu(target) || sched_idle_cpu(target)) &&
7028 : asym_fits_cpu(task_util, util_min, util_max, target))
7029 : return target;
7030 :
7031 : /*
7032 : * If the previous CPU is cache affine and idle, don't be stupid:
7033 : */
7034 : if (prev != target && cpus_share_cache(prev, target) &&
7035 : (available_idle_cpu(prev) || sched_idle_cpu(prev)) &&
7036 : asym_fits_cpu(task_util, util_min, util_max, prev))
7037 : return prev;
7038 :
7039 : /*
7040 : * Allow a per-cpu kthread to stack with the wakee if the
7041 : * kworker thread and the tasks previous CPUs are the same.
7042 : * The assumption is that the wakee queued work for the
7043 : * per-cpu kthread that is now complete and the wakeup is
7044 : * essentially a sync wakeup. An obvious example of this
7045 : * pattern is IO completions.
7046 : */
7047 : if (is_per_cpu_kthread(current) &&
7048 : in_task() &&
7049 : prev == smp_processor_id() &&
7050 : this_rq()->nr_running <= 1 &&
7051 : asym_fits_cpu(task_util, util_min, util_max, prev)) {
7052 : return prev;
7053 : }
7054 :
7055 : /* Check a recently used CPU as a potential idle candidate: */
7056 : recent_used_cpu = p->recent_used_cpu;
7057 : p->recent_used_cpu = prev;
7058 : if (recent_used_cpu != prev &&
7059 : recent_used_cpu != target &&
7060 : cpus_share_cache(recent_used_cpu, target) &&
7061 : (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) &&
7062 : cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr) &&
7063 : asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) {
7064 : return recent_used_cpu;
7065 : }
7066 :
7067 : /*
7068 : * For asymmetric CPU capacity systems, our domain of interest is
7069 : * sd_asym_cpucapacity rather than sd_llc.
7070 : */
7071 : if (sched_asym_cpucap_active()) {
7072 : sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target));
7073 : /*
7074 : * On an asymmetric CPU capacity system where an exclusive
7075 : * cpuset defines a symmetric island (i.e. one unique
7076 : * capacity_orig value through the cpuset), the key will be set
7077 : * but the CPUs within that cpuset will not have a domain with
7078 : * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric
7079 : * capacity path.
7080 : */
7081 : if (sd) {
7082 : i = select_idle_capacity(p, sd, target);
7083 : return ((unsigned)i < nr_cpumask_bits) ? i : target;
7084 : }
7085 : }
7086 :
7087 : sd = rcu_dereference(per_cpu(sd_llc, target));
7088 : if (!sd)
7089 : return target;
7090 :
7091 : if (sched_smt_active()) {
7092 : has_idle_core = test_idle_cores(target);
7093 :
7094 : if (!has_idle_core && cpus_share_cache(prev, target)) {
7095 : i = select_idle_smt(p, prev);
7096 : if ((unsigned int)i < nr_cpumask_bits)
7097 : return i;
7098 : }
7099 : }
7100 :
7101 : i = select_idle_cpu(p, sd, has_idle_core, target);
7102 : if ((unsigned)i < nr_cpumask_bits)
7103 : return i;
7104 :
7105 : return target;
7106 : }
7107 :
7108 : /*
7109 : * Predicts what cpu_util(@cpu) would return if @p was removed from @cpu
7110 : * (@dst_cpu = -1) or migrated to @dst_cpu.
7111 : */
7112 : static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu)
7113 : {
7114 : struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
7115 : unsigned long util = READ_ONCE(cfs_rq->avg.util_avg);
7116 :
7117 : /*
7118 : * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its
7119 : * contribution. If @p migrates from another CPU to @cpu add its
7120 : * contribution. In all the other cases @cpu is not impacted by the
7121 : * migration so its util_avg is already correct.
7122 : */
7123 : if (task_cpu(p) == cpu && dst_cpu != cpu)
7124 : lsub_positive(&util, task_util(p));
7125 : else if (task_cpu(p) != cpu && dst_cpu == cpu)
7126 : util += task_util(p);
7127 :
7128 : if (sched_feat(UTIL_EST)) {
7129 : unsigned long util_est;
7130 :
7131 : util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);
7132 :
7133 : /*
7134 : * During wake-up @p isn't enqueued yet and doesn't contribute
7135 : * to any cpu_rq(cpu)->cfs.avg.util_est.enqueued.
7136 : * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p
7137 : * has been enqueued.
7138 : *
7139 : * During exec (@dst_cpu = -1) @p is enqueued and does
7140 : * contribute to cpu_rq(cpu)->cfs.util_est.enqueued.
7141 : * Remove it to "simulate" cpu_util without @p's contribution.
7142 : *
7143 : * Despite the task_on_rq_queued(@p) check there is still a
7144 : * small window for a possible race when an exec
7145 : * select_task_rq_fair() races with LB's detach_task().
7146 : *
7147 : * detach_task()
7148 : * deactivate_task()
7149 : * p->on_rq = TASK_ON_RQ_MIGRATING;
7150 : * -------------------------------- A
7151 : * dequeue_task() \
7152 : * dequeue_task_fair() + Race Time
7153 : * util_est_dequeue() /
7154 : * -------------------------------- B
7155 : *
7156 : * The additional check "current == p" is required to further
7157 : * reduce the race window.
7158 : */
7159 : if (dst_cpu == cpu)
7160 : util_est += _task_util_est(p);
7161 : else if (unlikely(task_on_rq_queued(p) || current == p))
7162 : lsub_positive(&util_est, _task_util_est(p));
7163 :
7164 : util = max(util, util_est);
7165 : }
7166 :
7167 : return min(util, capacity_orig_of(cpu));
7168 : }
7169 :
7170 : /*
7171 : * cpu_util_without: compute cpu utilization without any contributions from *p
7172 : * @cpu: the CPU which utilization is requested
7173 : * @p: the task which utilization should be discounted
7174 : *
7175 : * The utilization of a CPU is defined by the utilization of tasks currently
7176 : * enqueued on that CPU as well as tasks which are currently sleeping after an
7177 : * execution on that CPU.
7178 : *
7179 : * This method returns the utilization of the specified CPU by discounting the
7180 : * utilization of the specified task, whenever the task is currently
7181 : * contributing to the CPU utilization.
7182 : */
7183 : static unsigned long cpu_util_without(int cpu, struct task_struct *p)
7184 : {
7185 : /* Task has no contribution or is new */
7186 : if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
7187 : return cpu_util_cfs(cpu);
7188 :
7189 : return cpu_util_next(cpu, p, -1);
7190 : }
7191 :
7192 : /*
7193 : * energy_env - Utilization landscape for energy estimation.
7194 : * @task_busy_time: Utilization contribution by the task for which we test the
7195 : * placement. Given by eenv_task_busy_time().
7196 : * @pd_busy_time: Utilization of the whole perf domain without the task
7197 : * contribution. Given by eenv_pd_busy_time().
7198 : * @cpu_cap: Maximum CPU capacity for the perf domain.
7199 : * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap).
7200 : */
7201 : struct energy_env {
7202 : unsigned long task_busy_time;
7203 : unsigned long pd_busy_time;
7204 : unsigned long cpu_cap;
7205 : unsigned long pd_cap;
7206 : };
7207 :
7208 : /*
7209 : * Compute the task busy time for compute_energy(). This time cannot be
7210 : * injected directly into effective_cpu_util() because of the IRQ scaling.
7211 : * The latter only makes sense with the most recent CPUs where the task has
7212 : * run.
7213 : */
7214 : static inline void eenv_task_busy_time(struct energy_env *eenv,
7215 : struct task_struct *p, int prev_cpu)
7216 : {
7217 : unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu);
7218 : unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu));
7219 :
7220 : if (unlikely(irq >= max_cap))
7221 : busy_time = max_cap;
7222 : else
7223 : busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap);
7224 :
7225 : eenv->task_busy_time = busy_time;
7226 : }
7227 :
7228 : /*
7229 : * Compute the perf_domain (PD) busy time for compute_energy(). Based on the
7230 : * utilization for each @pd_cpus, it however doesn't take into account
7231 : * clamping since the ratio (utilization / cpu_capacity) is already enough to
7232 : * scale the EM reported power consumption at the (eventually clamped)
7233 : * cpu_capacity.
7234 : *
7235 : * The contribution of the task @p for which we want to estimate the
7236 : * energy cost is removed (by cpu_util_next()) and must be calculated
7237 : * separately (see eenv_task_busy_time). This ensures:
7238 : *
7239 : * - A stable PD utilization, no matter which CPU of that PD we want to place
7240 : * the task on.
7241 : *
7242 : * - A fair comparison between CPUs as the task contribution (task_util())
7243 : * will always be the same no matter which CPU utilization we rely on
7244 : * (util_avg or util_est).
7245 : *
7246 : * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't
7247 : * exceed @eenv->pd_cap.
7248 : */
7249 : static inline void eenv_pd_busy_time(struct energy_env *eenv,
7250 : struct cpumask *pd_cpus,
7251 : struct task_struct *p)
7252 : {
7253 : unsigned long busy_time = 0;
7254 : int cpu;
7255 :
7256 : for_each_cpu(cpu, pd_cpus) {
7257 : unsigned long util = cpu_util_next(cpu, p, -1);
7258 :
7259 : busy_time += effective_cpu_util(cpu, util, ENERGY_UTIL, NULL);
7260 : }
7261 :
7262 : eenv->pd_busy_time = min(eenv->pd_cap, busy_time);
7263 : }
7264 :
7265 : /*
7266 : * Compute the maximum utilization for compute_energy() when the task @p
7267 : * is placed on the cpu @dst_cpu.
7268 : *
7269 : * Returns the maximum utilization among @eenv->cpus. This utilization can't
7270 : * exceed @eenv->cpu_cap.
7271 : */
7272 : static inline unsigned long
7273 : eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus,
7274 : struct task_struct *p, int dst_cpu)
7275 : {
7276 : unsigned long max_util = 0;
7277 : int cpu;
7278 :
7279 : for_each_cpu(cpu, pd_cpus) {
7280 : struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL;
7281 : unsigned long util = cpu_util_next(cpu, p, dst_cpu);
7282 : unsigned long cpu_util;
7283 :
7284 : /*
7285 : * Performance domain frequency: utilization clamping
7286 : * must be considered since it affects the selection
7287 : * of the performance domain frequency.
7288 : * NOTE: in case RT tasks are running, by default the
7289 : * FREQUENCY_UTIL's utilization can be max OPP.
7290 : */
7291 : cpu_util = effective_cpu_util(cpu, util, FREQUENCY_UTIL, tsk);
7292 : max_util = max(max_util, cpu_util);
7293 : }
7294 :
7295 : return min(max_util, eenv->cpu_cap);
7296 : }
7297 :
7298 : /*
7299 : * compute_energy(): Use the Energy Model to estimate the energy that @pd would
7300 : * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task
7301 : * contribution is ignored.
7302 : */
7303 : static inline unsigned long
7304 : compute_energy(struct energy_env *eenv, struct perf_domain *pd,
7305 : struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu)
7306 : {
7307 : unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu);
7308 : unsigned long busy_time = eenv->pd_busy_time;
7309 :
7310 : if (dst_cpu >= 0)
7311 : busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time);
7312 :
7313 : return em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap);
7314 : }
7315 :
7316 : /*
7317 : * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the
7318 : * waking task. find_energy_efficient_cpu() looks for the CPU with maximum
7319 : * spare capacity in each performance domain and uses it as a potential
7320 : * candidate to execute the task. Then, it uses the Energy Model to figure
7321 : * out which of the CPU candidates is the most energy-efficient.
7322 : *
7323 : * The rationale for this heuristic is as follows. In a performance domain,
7324 : * all the most energy efficient CPU candidates (according to the Energy
7325 : * Model) are those for which we'll request a low frequency. When there are
7326 : * several CPUs for which the frequency request will be the same, we don't
7327 : * have enough data to break the tie between them, because the Energy Model
7328 : * only includes active power costs. With this model, if we assume that
7329 : * frequency requests follow utilization (e.g. using schedutil), the CPU with
7330 : * the maximum spare capacity in a performance domain is guaranteed to be among
7331 : * the best candidates of the performance domain.
7332 : *
7333 : * In practice, it could be preferable from an energy standpoint to pack
7334 : * small tasks on a CPU in order to let other CPUs go in deeper idle states,
7335 : * but that could also hurt our chances to go cluster idle, and we have no
7336 : * ways to tell with the current Energy Model if this is actually a good
7337 : * idea or not. So, find_energy_efficient_cpu() basically favors
7338 : * cluster-packing, and spreading inside a cluster. That should at least be
7339 : * a good thing for latency, and this is consistent with the idea that most
7340 : * of the energy savings of EAS come from the asymmetry of the system, and
7341 : * not so much from breaking the tie between identical CPUs. That's also the
7342 : * reason why EAS is enabled in the topology code only for systems where
7343 : * SD_ASYM_CPUCAPACITY is set.
7344 : *
7345 : * NOTE: Forkees are not accepted in the energy-aware wake-up path because
7346 : * they don't have any useful utilization data yet and it's not possible to
7347 : * forecast their impact on energy consumption. Consequently, they will be
7348 : * placed by find_idlest_cpu() on the least loaded CPU, which might turn out
7349 : * to be energy-inefficient in some use-cases. The alternative would be to
7350 : * bias new tasks towards specific types of CPUs first, or to try to infer
7351 : * their util_avg from the parent task, but those heuristics could hurt
7352 : * other use-cases too. So, until someone finds a better way to solve this,
7353 : * let's keep things simple by re-using the existing slow path.
7354 : */
7355 : static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
7356 : {
7357 : struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
7358 : unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
7359 : unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0;
7360 : unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024;
7361 : struct root_domain *rd = this_rq()->rd;
7362 : int cpu, best_energy_cpu, target = -1;
7363 : int prev_fits = -1, best_fits = -1;
7364 : unsigned long best_thermal_cap = 0;
7365 : unsigned long prev_thermal_cap = 0;
7366 : struct sched_domain *sd;
7367 : struct perf_domain *pd;
7368 : struct energy_env eenv;
7369 :
7370 : rcu_read_lock();
7371 : pd = rcu_dereference(rd->pd);
7372 : if (!pd || READ_ONCE(rd->overutilized))
7373 : goto unlock;
7374 :
7375 : /*
7376 : * Energy-aware wake-up happens on the lowest sched_domain starting
7377 : * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu.
7378 : */
7379 : sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity));
7380 : while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
7381 : sd = sd->parent;
7382 : if (!sd)
7383 : goto unlock;
7384 :
7385 : target = prev_cpu;
7386 :
7387 : sync_entity_load_avg(&p->se);
7388 : if (!uclamp_task_util(p, p_util_min, p_util_max))
7389 : goto unlock;
7390 :
7391 : eenv_task_busy_time(&eenv, p, prev_cpu);
7392 :
7393 : for (; pd; pd = pd->next) {
7394 : unsigned long util_min = p_util_min, util_max = p_util_max;
7395 : unsigned long cpu_cap, cpu_thermal_cap, util;
7396 : unsigned long cur_delta, max_spare_cap = 0;
7397 : unsigned long rq_util_min, rq_util_max;
7398 : unsigned long prev_spare_cap = 0;
7399 : int max_spare_cap_cpu = -1;
7400 : unsigned long base_energy;
7401 : int fits, max_fits = -1;
7402 :
7403 : cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask);
7404 :
7405 : if (cpumask_empty(cpus))
7406 : continue;
7407 :
7408 : /* Account thermal pressure for the energy estimation */
7409 : cpu = cpumask_first(cpus);
7410 : cpu_thermal_cap = arch_scale_cpu_capacity(cpu);
7411 : cpu_thermal_cap -= arch_scale_thermal_pressure(cpu);
7412 :
7413 : eenv.cpu_cap = cpu_thermal_cap;
7414 : eenv.pd_cap = 0;
7415 :
7416 : for_each_cpu(cpu, cpus) {
7417 : struct rq *rq = cpu_rq(cpu);
7418 :
7419 : eenv.pd_cap += cpu_thermal_cap;
7420 :
7421 : if (!cpumask_test_cpu(cpu, sched_domain_span(sd)))
7422 : continue;
7423 :
7424 : if (!cpumask_test_cpu(cpu, p->cpus_ptr))
7425 : continue;
7426 :
7427 : util = cpu_util_next(cpu, p, cpu);
7428 : cpu_cap = capacity_of(cpu);
7429 :
7430 : /*
7431 : * Skip CPUs that cannot satisfy the capacity request.
7432 : * IOW, placing the task there would make the CPU
7433 : * overutilized. Take uclamp into account to see how
7434 : * much capacity we can get out of the CPU; this is
7435 : * aligned with sched_cpu_util().
7436 : */
7437 : if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) {
7438 : /*
7439 : * Open code uclamp_rq_util_with() except for
7440 : * the clamp() part. Ie: apply max aggregation
7441 : * only. util_fits_cpu() logic requires to
7442 : * operate on non clamped util but must use the
7443 : * max-aggregated uclamp_{min, max}.
7444 : */
7445 : rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN);
7446 : rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX);
7447 :
7448 : util_min = max(rq_util_min, p_util_min);
7449 : util_max = max(rq_util_max, p_util_max);
7450 : }
7451 :
7452 : fits = util_fits_cpu(util, util_min, util_max, cpu);
7453 : if (!fits)
7454 : continue;
7455 :
7456 : lsub_positive(&cpu_cap, util);
7457 :
7458 : if (cpu == prev_cpu) {
7459 : /* Always use prev_cpu as a candidate. */
7460 : prev_spare_cap = cpu_cap;
7461 : prev_fits = fits;
7462 : } else if ((fits > max_fits) ||
7463 : ((fits == max_fits) && (cpu_cap > max_spare_cap))) {
7464 : /*
7465 : * Find the CPU with the maximum spare capacity
7466 : * among the remaining CPUs in the performance
7467 : * domain.
7468 : */
7469 : max_spare_cap = cpu_cap;
7470 : max_spare_cap_cpu = cpu;
7471 : max_fits = fits;
7472 : }
7473 : }
7474 :
7475 : if (max_spare_cap_cpu < 0 && prev_spare_cap == 0)
7476 : continue;
7477 :
7478 : eenv_pd_busy_time(&eenv, cpus, p);
7479 : /* Compute the 'base' energy of the pd, without @p */
7480 : base_energy = compute_energy(&eenv, pd, cpus, p, -1);
7481 :
7482 : /* Evaluate the energy impact of using prev_cpu. */
7483 : if (prev_spare_cap > 0) {
7484 : prev_delta = compute_energy(&eenv, pd, cpus, p,
7485 : prev_cpu);
7486 : /* CPU utilization has changed */
7487 : if (prev_delta < base_energy)
7488 : goto unlock;
7489 : prev_delta -= base_energy;
7490 : prev_thermal_cap = cpu_thermal_cap;
7491 : best_delta = min(best_delta, prev_delta);
7492 : }
7493 :
7494 : /* Evaluate the energy impact of using max_spare_cap_cpu. */
7495 : if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) {
7496 : /* Current best energy cpu fits better */
7497 : if (max_fits < best_fits)
7498 : continue;
7499 :
7500 : /*
7501 : * Both don't fit performance hint (i.e. uclamp_min)
7502 : * but best energy cpu has better capacity.
7503 : */
7504 : if ((max_fits < 0) &&
7505 : (cpu_thermal_cap <= best_thermal_cap))
7506 : continue;
7507 :
7508 : cur_delta = compute_energy(&eenv, pd, cpus, p,
7509 : max_spare_cap_cpu);
7510 : /* CPU utilization has changed */
7511 : if (cur_delta < base_energy)
7512 : goto unlock;
7513 : cur_delta -= base_energy;
7514 :
7515 : /*
7516 : * Both fit for the task but best energy cpu has lower
7517 : * energy impact.
7518 : */
7519 : if ((max_fits > 0) && (best_fits > 0) &&
7520 : (cur_delta >= best_delta))
7521 : continue;
7522 :
7523 : best_delta = cur_delta;
7524 : best_energy_cpu = max_spare_cap_cpu;
7525 : best_fits = max_fits;
7526 : best_thermal_cap = cpu_thermal_cap;
7527 : }
7528 : }
7529 : rcu_read_unlock();
7530 :
7531 : if ((best_fits > prev_fits) ||
7532 : ((best_fits > 0) && (best_delta < prev_delta)) ||
7533 : ((best_fits < 0) && (best_thermal_cap > prev_thermal_cap)))
7534 : target = best_energy_cpu;
7535 :
7536 : return target;
7537 :
7538 : unlock:
7539 : rcu_read_unlock();
7540 :
7541 : return target;
7542 : }
7543 :
7544 : /*
7545 : * select_task_rq_fair: Select target runqueue for the waking task in domains
7546 : * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE,
7547 : * SD_BALANCE_FORK, or SD_BALANCE_EXEC.
7548 : *
7549 : * Balances load by selecting the idlest CPU in the idlest group, or under
7550 : * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set.
7551 : *
7552 : * Returns the target CPU number.
7553 : */
7554 : static int
7555 : select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
7556 : {
7557 : int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING);
7558 : struct sched_domain *tmp, *sd = NULL;
7559 : int cpu = smp_processor_id();
7560 : int new_cpu = prev_cpu;
7561 : int want_affine = 0;
7562 : /* SD_flags and WF_flags share the first nibble */
7563 : int sd_flag = wake_flags & 0xF;
7564 :
7565 : /*
7566 : * required for stable ->cpus_allowed
7567 : */
7568 : lockdep_assert_held(&p->pi_lock);
7569 : if (wake_flags & WF_TTWU) {
7570 : record_wakee(p);
7571 :
7572 : if (sched_energy_enabled()) {
7573 : new_cpu = find_energy_efficient_cpu(p, prev_cpu);
7574 : if (new_cpu >= 0)
7575 : return new_cpu;
7576 : new_cpu = prev_cpu;
7577 : }
7578 :
7579 : want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr);
7580 : }
7581 :
7582 : rcu_read_lock();
7583 : for_each_domain(cpu, tmp) {
7584 : /*
7585 : * If both 'cpu' and 'prev_cpu' are part of this domain,
7586 : * cpu is a valid SD_WAKE_AFFINE target.
7587 : */
7588 : if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
7589 : cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
7590 : if (cpu != prev_cpu)
7591 : new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync);
7592 :
7593 : sd = NULL; /* Prefer wake_affine over balance flags */
7594 : break;
7595 : }
7596 :
7597 : /*
7598 : * Usually only true for WF_EXEC and WF_FORK, as sched_domains
7599 : * usually do not have SD_BALANCE_WAKE set. That means wakeup
7600 : * will usually go to the fast path.
7601 : */
7602 : if (tmp->flags & sd_flag)
7603 : sd = tmp;
7604 : else if (!want_affine)
7605 : break;
7606 : }
7607 :
7608 : if (unlikely(sd)) {
7609 : /* Slow path */
7610 : new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag);
7611 : } else if (wake_flags & WF_TTWU) { /* XXX always ? */
7612 : /* Fast path */
7613 : new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);
7614 : }
7615 : rcu_read_unlock();
7616 :
7617 : return new_cpu;
7618 : }
7619 :
7620 : /*
7621 : * Called immediately before a task is migrated to a new CPU; task_cpu(p) and
7622 : * cfs_rq_of(p) references at time of call are still valid and identify the
7623 : * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held.
7624 : */
7625 : static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)
7626 : {
7627 : struct sched_entity *se = &p->se;
7628 :
7629 : /*
7630 : * As blocked tasks retain absolute vruntime the migration needs to
7631 : * deal with this by subtracting the old and adding the new
7632 : * min_vruntime -- the latter is done by enqueue_entity() when placing
7633 : * the task on the new runqueue.
7634 : */
7635 : if (READ_ONCE(p->__state) == TASK_WAKING) {
7636 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
7637 :
7638 : se->vruntime -= u64_u32_load(cfs_rq->min_vruntime);
7639 : }
7640 :
7641 : if (!task_on_rq_migrating(p)) {
7642 : remove_entity_load_avg(se);
7643 :
7644 : /*
7645 : * Here, the task's PELT values have been updated according to
7646 : * the current rq's clock. But if that clock hasn't been
7647 : * updated in a while, a substantial idle time will be missed,
7648 : * leading to an inflation after wake-up on the new rq.
7649 : *
7650 : * Estimate the missing time from the cfs_rq last_update_time
7651 : * and update sched_avg to improve the PELT continuity after
7652 : * migration.
7653 : */
7654 : migrate_se_pelt_lag(se);
7655 : }
7656 :
7657 : /* Tell new CPU we are migrated */
7658 : se->avg.last_update_time = 0;
7659 :
7660 : /* We have migrated, no longer consider this task hot */
7661 : se->exec_start = 0;
7662 :
7663 : update_scan_period(p, new_cpu);
7664 : }
7665 :
7666 : static void task_dead_fair(struct task_struct *p)
7667 : {
7668 : remove_entity_load_avg(&p->se);
7669 : }
7670 :
7671 : static int
7672 : balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
7673 : {
7674 : if (rq->nr_running)
7675 : return 1;
7676 :
7677 : return newidle_balance(rq, rf) != 0;
7678 : }
7679 : #endif /* CONFIG_SMP */
7680 :
7681 : static unsigned long wakeup_gran(struct sched_entity *se)
7682 : {
7683 750 : unsigned long gran = sysctl_sched_wakeup_granularity;
7684 :
7685 : /*
7686 : * Since its curr running now, convert the gran from real-time
7687 : * to virtual-time in his units.
7688 : *
7689 : * By using 'se' instead of 'curr' we penalize light tasks, so
7690 : * they get preempted easier. That is, if 'se' < 'curr' then
7691 : * the resulting gran will be larger, therefore penalizing the
7692 : * lighter, if otoh 'se' > 'curr' then the resulting gran will
7693 : * be smaller, again penalizing the lighter task.
7694 : *
7695 : * This is especially important for buddies when the leftmost
7696 : * task is higher priority than the buddy.
7697 : */
7698 750 : return calc_delta_fair(gran, se);
7699 : }
7700 :
7701 : /*
7702 : * Should 'se' preempt 'curr'.
7703 : *
7704 : * |s1
7705 : * |s2
7706 : * |s3
7707 : * g
7708 : * |<--->|c
7709 : *
7710 : * w(c, s1) = -1
7711 : * w(c, s2) = 0
7712 : * w(c, s3) = 1
7713 : *
7714 : */
7715 : static int
7716 2525 : wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
7717 : {
7718 2525 : s64 gran, vdiff = curr->vruntime - se->vruntime;
7719 :
7720 2525 : if (vdiff <= 0)
7721 : return -1;
7722 :
7723 750 : gran = wakeup_gran(se);
7724 750 : if (vdiff > gran)
7725 : return 1;
7726 :
7727 : return 0;
7728 : }
7729 :
7730 0 : static void set_last_buddy(struct sched_entity *se)
7731 : {
7732 0 : for_each_sched_entity(se) {
7733 0 : if (SCHED_WARN_ON(!se->on_rq))
7734 : return;
7735 0 : if (se_is_idle(se))
7736 : return;
7737 0 : cfs_rq_of(se)->last = se;
7738 : }
7739 : }
7740 :
7741 746 : static void set_next_buddy(struct sched_entity *se)
7742 : {
7743 1492 : for_each_sched_entity(se) {
7744 746 : if (SCHED_WARN_ON(!se->on_rq))
7745 : return;
7746 746 : if (se_is_idle(se))
7747 : return;
7748 1492 : cfs_rq_of(se)->next = se;
7749 : }
7750 : }
7751 :
7752 : static void set_skip_buddy(struct sched_entity *se)
7753 : {
7754 0 : for_each_sched_entity(se)
7755 0 : cfs_rq_of(se)->skip = se;
7756 : }
7757 :
7758 : /*
7759 : * Preempt the current task with a newly woken task if needed:
7760 : */
7761 2125 : static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
7762 : {
7763 2125 : struct task_struct *curr = rq->curr;
7764 2125 : struct sched_entity *se = &curr->se, *pse = &p->se;
7765 4250 : struct cfs_rq *cfs_rq = task_cfs_rq(curr);
7766 2125 : int scale = cfs_rq->nr_running >= sched_nr_latency;
7767 2125 : int next_buddy_marked = 0;
7768 : int cse_is_idle, pse_is_idle;
7769 :
7770 2125 : if (unlikely(se == pse))
7771 : return;
7772 :
7773 : /*
7774 : * This is possible from callers such as attach_tasks(), in which we
7775 : * unconditionally check_preempt_curr() after an enqueue (which may have
7776 : * lead to a throttle). This both saves work and prevents false
7777 : * next-buddy nomination below.
7778 : */
7779 2125 : if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
7780 : return;
7781 :
7782 2125 : if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
7783 0 : set_next_buddy(pse);
7784 0 : next_buddy_marked = 1;
7785 : }
7786 :
7787 : /*
7788 : * We can come here with TIF_NEED_RESCHED already set from new task
7789 : * wake up path.
7790 : *
7791 : * Note: this also catches the edge-case of curr being in a throttled
7792 : * group (e.g. via set_curr_task), since update_curr() (in the
7793 : * enqueue of curr) will have resulted in resched being set. This
7794 : * prevents us from potentially nominating it as a false LAST_BUDDY
7795 : * below.
7796 : */
7797 2125 : if (test_tsk_need_resched(curr))
7798 : return;
7799 :
7800 : /* Idle tasks are by definition preempted by non-idle tasks. */
7801 3558 : if (unlikely(task_has_idle_policy(curr)) &&
7802 0 : likely(!task_has_idle_policy(p)))
7803 : goto preempt;
7804 :
7805 : /*
7806 : * Batch and idle tasks do not preempt non-idle tasks (their preemption
7807 : * is driven by the tick):
7808 : */
7809 1779 : if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
7810 : return;
7811 :
7812 1779 : find_matching_se(&se, &pse);
7813 1779 : WARN_ON_ONCE(!pse);
7814 :
7815 1779 : cse_is_idle = se_is_idle(se);
7816 1779 : pse_is_idle = se_is_idle(pse);
7817 :
7818 : /*
7819 : * Preempt an idle group in favor of a non-idle group (and don't preempt
7820 : * in the inverse case).
7821 : */
7822 : if (cse_is_idle && !pse_is_idle)
7823 : goto preempt;
7824 : if (cse_is_idle != pse_is_idle)
7825 : return;
7826 :
7827 3558 : update_curr(cfs_rq_of(se));
7828 1779 : if (wakeup_preempt_entity(se, pse) == 1) {
7829 : /*
7830 : * Bias pick_next to pick the sched entity that is
7831 : * triggering this preemption.
7832 : */
7833 746 : if (!next_buddy_marked)
7834 746 : set_next_buddy(pse);
7835 : goto preempt;
7836 : }
7837 :
7838 : return;
7839 :
7840 : preempt:
7841 746 : resched_curr(rq);
7842 : /*
7843 : * Only set the backward buddy when the current task is still
7844 : * on the rq. This can happen when a wakeup gets interleaved
7845 : * with schedule on the ->pre_schedule() or idle_balance()
7846 : * point, either of which can * drop the rq lock.
7847 : *
7848 : * Also, during early boot the idle thread is in the fair class,
7849 : * for obvious reasons its a bad idea to schedule back to it.
7850 : */
7851 746 : if (unlikely(!se->on_rq || curr == rq->idle))
7852 : return;
7853 :
7854 746 : if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
7855 0 : set_last_buddy(se);
7856 : }
7857 :
7858 : #ifdef CONFIG_SMP
7859 : static struct task_struct *pick_task_fair(struct rq *rq)
7860 : {
7861 : struct sched_entity *se;
7862 : struct cfs_rq *cfs_rq;
7863 :
7864 : again:
7865 : cfs_rq = &rq->cfs;
7866 : if (!cfs_rq->nr_running)
7867 : return NULL;
7868 :
7869 : do {
7870 : struct sched_entity *curr = cfs_rq->curr;
7871 :
7872 : /* When we pick for a remote RQ, we'll not have done put_prev_entity() */
7873 : if (curr) {
7874 : if (curr->on_rq)
7875 : update_curr(cfs_rq);
7876 : else
7877 : curr = NULL;
7878 :
7879 : if (unlikely(check_cfs_rq_runtime(cfs_rq)))
7880 : goto again;
7881 : }
7882 :
7883 : se = pick_next_entity(cfs_rq, curr);
7884 : cfs_rq = group_cfs_rq(se);
7885 : } while (cfs_rq);
7886 :
7887 : return task_of(se);
7888 : }
7889 : #endif
7890 :
7891 : struct task_struct *
7892 2210 : pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
7893 : {
7894 2210 : struct cfs_rq *cfs_rq = &rq->cfs;
7895 : struct sched_entity *se;
7896 : struct task_struct *p;
7897 : int new_tasks;
7898 :
7899 : again:
7900 2210 : if (!sched_fair_runnable(rq))
7901 : goto idle;
7902 :
7903 : #ifdef CONFIG_FAIR_GROUP_SCHED
7904 : if (!prev || prev->sched_class != &fair_sched_class)
7905 : goto simple;
7906 :
7907 : /*
7908 : * Because of the set_next_buddy() in dequeue_task_fair() it is rather
7909 : * likely that a next task is from the same cgroup as the current.
7910 : *
7911 : * Therefore attempt to avoid putting and setting the entire cgroup
7912 : * hierarchy, only change the part that actually changes.
7913 : */
7914 :
7915 : do {
7916 : struct sched_entity *curr = cfs_rq->curr;
7917 :
7918 : /*
7919 : * Since we got here without doing put_prev_entity() we also
7920 : * have to consider cfs_rq->curr. If it is still a runnable
7921 : * entity, update_curr() will update its vruntime, otherwise
7922 : * forget we've ever seen it.
7923 : */
7924 : if (curr) {
7925 : if (curr->on_rq)
7926 : update_curr(cfs_rq);
7927 : else
7928 : curr = NULL;
7929 :
7930 : /*
7931 : * This call to check_cfs_rq_runtime() will do the
7932 : * throttle and dequeue its entity in the parent(s).
7933 : * Therefore the nr_running test will indeed
7934 : * be correct.
7935 : */
7936 : if (unlikely(check_cfs_rq_runtime(cfs_rq))) {
7937 : cfs_rq = &rq->cfs;
7938 :
7939 : if (!cfs_rq->nr_running)
7940 : goto idle;
7941 :
7942 : goto simple;
7943 : }
7944 : }
7945 :
7946 : se = pick_next_entity(cfs_rq, curr);
7947 : cfs_rq = group_cfs_rq(se);
7948 : } while (cfs_rq);
7949 :
7950 : p = task_of(se);
7951 :
7952 : /*
7953 : * Since we haven't yet done put_prev_entity and if the selected task
7954 : * is a different task than we started out with, try and touch the
7955 : * least amount of cfs_rqs.
7956 : */
7957 : if (prev != p) {
7958 : struct sched_entity *pse = &prev->se;
7959 :
7960 : while (!(cfs_rq = is_same_group(se, pse))) {
7961 : int se_depth = se->depth;
7962 : int pse_depth = pse->depth;
7963 :
7964 : if (se_depth <= pse_depth) {
7965 : put_prev_entity(cfs_rq_of(pse), pse);
7966 : pse = parent_entity(pse);
7967 : }
7968 : if (se_depth >= pse_depth) {
7969 : set_next_entity(cfs_rq_of(se), se);
7970 : se = parent_entity(se);
7971 : }
7972 : }
7973 :
7974 : put_prev_entity(cfs_rq, pse);
7975 : set_next_entity(cfs_rq, se);
7976 : }
7977 :
7978 : goto done;
7979 : simple:
7980 : #endif
7981 2209 : if (prev)
7982 2209 : put_prev_task(rq, prev);
7983 :
7984 : do {
7985 2209 : se = pick_next_entity(cfs_rq, NULL);
7986 2209 : set_next_entity(cfs_rq, se);
7987 2209 : cfs_rq = group_cfs_rq(se);
7988 : } while (cfs_rq);
7989 :
7990 2209 : p = task_of(se);
7991 :
7992 : done: __maybe_unused;
7993 : #ifdef CONFIG_SMP
7994 : /*
7995 : * Move the next running task to the front of
7996 : * the list, so our cfs_tasks list becomes MRU
7997 : * one.
7998 : */
7999 : list_move(&p->se.group_node, &rq->cfs_tasks);
8000 : #endif
8001 :
8002 2209 : if (hrtick_enabled_fair(rq))
8003 : hrtick_start_fair(rq, p);
8004 :
8005 2209 : update_misfit_status(p, rq);
8006 :
8007 2209 : return p;
8008 :
8009 : idle:
8010 : if (!rf)
8011 : return NULL;
8012 :
8013 : new_tasks = newidle_balance(rq, rf);
8014 :
8015 : /*
8016 : * Because newidle_balance() releases (and re-acquires) rq->lock, it is
8017 : * possible for any higher priority task to appear. In that case we
8018 : * must re-start the pick_next_entity() loop.
8019 : */
8020 : if (new_tasks < 0)
8021 : return RETRY_TASK;
8022 :
8023 : if (new_tasks > 0)
8024 : goto again;
8025 :
8026 : /*
8027 : * rq is about to be idle, check if we need to update the
8028 : * lost_idle_time of clock_pelt
8029 : */
8030 : update_idle_rq_clock_pelt(rq);
8031 :
8032 : return NULL;
8033 : }
8034 :
8035 0 : static struct task_struct *__pick_next_task_fair(struct rq *rq)
8036 : {
8037 0 : return pick_next_task_fair(rq, NULL, NULL);
8038 : }
8039 :
8040 : /*
8041 : * Account for a descheduled task:
8042 : */
8043 2212 : static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
8044 : {
8045 2212 : struct sched_entity *se = &prev->se;
8046 : struct cfs_rq *cfs_rq;
8047 :
8048 4424 : for_each_sched_entity(se) {
8049 4424 : cfs_rq = cfs_rq_of(se);
8050 2212 : put_prev_entity(cfs_rq, se);
8051 : }
8052 2212 : }
8053 :
8054 : /*
8055 : * sched_yield() is very simple
8056 : *
8057 : * The magic of dealing with the ->skip buddy is in pick_next_entity.
8058 : */
8059 0 : static void yield_task_fair(struct rq *rq)
8060 : {
8061 0 : struct task_struct *curr = rq->curr;
8062 0 : struct cfs_rq *cfs_rq = task_cfs_rq(curr);
8063 0 : struct sched_entity *se = &curr->se;
8064 :
8065 : /*
8066 : * Are we the only task in the tree?
8067 : */
8068 0 : if (unlikely(rq->nr_running == 1))
8069 : return;
8070 :
8071 0 : clear_buddies(cfs_rq, se);
8072 :
8073 0 : if (curr->policy != SCHED_BATCH) {
8074 0 : update_rq_clock(rq);
8075 : /*
8076 : * Update run-time statistics of the 'current'.
8077 : */
8078 0 : update_curr(cfs_rq);
8079 : /*
8080 : * Tell update_rq_clock() that we've just updated,
8081 : * so we don't do microscopic update in schedule()
8082 : * and double the fastpath cost.
8083 : */
8084 0 : rq_clock_skip_update(rq);
8085 : }
8086 :
8087 : set_skip_buddy(se);
8088 : }
8089 :
8090 0 : static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
8091 : {
8092 0 : struct sched_entity *se = &p->se;
8093 :
8094 : /* throttled hierarchies are not runnable */
8095 0 : if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se)))
8096 : return false;
8097 :
8098 : /* Tell the scheduler that we'd really like pse to run next. */
8099 0 : set_next_buddy(se);
8100 :
8101 0 : yield_task_fair(rq);
8102 :
8103 0 : return true;
8104 : }
8105 :
8106 : #ifdef CONFIG_SMP
8107 : /**************************************************
8108 : * Fair scheduling class load-balancing methods.
8109 : *
8110 : * BASICS
8111 : *
8112 : * The purpose of load-balancing is to achieve the same basic fairness the
8113 : * per-CPU scheduler provides, namely provide a proportional amount of compute
8114 : * time to each task. This is expressed in the following equation:
8115 : *
8116 : * W_i,n/P_i == W_j,n/P_j for all i,j (1)
8117 : *
8118 : * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight
8119 : * W_i,0 is defined as:
8120 : *
8121 : * W_i,0 = \Sum_j w_i,j (2)
8122 : *
8123 : * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight
8124 : * is derived from the nice value as per sched_prio_to_weight[].
8125 : *
8126 : * The weight average is an exponential decay average of the instantaneous
8127 : * weight:
8128 : *
8129 : * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3)
8130 : *
8131 : * C_i is the compute capacity of CPU i, typically it is the
8132 : * fraction of 'recent' time available for SCHED_OTHER task execution. But it
8133 : * can also include other factors [XXX].
8134 : *
8135 : * To achieve this balance we define a measure of imbalance which follows
8136 : * directly from (1):
8137 : *
8138 : * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4)
8139 : *
8140 : * We them move tasks around to minimize the imbalance. In the continuous
8141 : * function space it is obvious this converges, in the discrete case we get
8142 : * a few fun cases generally called infeasible weight scenarios.
8143 : *
8144 : * [XXX expand on:
8145 : * - infeasible weights;
8146 : * - local vs global optima in the discrete case. ]
8147 : *
8148 : *
8149 : * SCHED DOMAINS
8150 : *
8151 : * In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
8152 : * for all i,j solution, we create a tree of CPUs that follows the hardware
8153 : * topology where each level pairs two lower groups (or better). This results
8154 : * in O(log n) layers. Furthermore we reduce the number of CPUs going up the
8155 : * tree to only the first of the previous level and we decrease the frequency
8156 : * of load-balance at each level inv. proportional to the number of CPUs in
8157 : * the groups.
8158 : *
8159 : * This yields:
8160 : *
8161 : * log_2 n 1 n
8162 : * \Sum { --- * --- * 2^i } = O(n) (5)
8163 : * i = 0 2^i 2^i
8164 : * `- size of each group
8165 : * | | `- number of CPUs doing load-balance
8166 : * | `- freq
8167 : * `- sum over all levels
8168 : *
8169 : * Coupled with a limit on how many tasks we can migrate every balance pass,
8170 : * this makes (5) the runtime complexity of the balancer.
8171 : *
8172 : * An important property here is that each CPU is still (indirectly) connected
8173 : * to every other CPU in at most O(log n) steps:
8174 : *
8175 : * The adjacency matrix of the resulting graph is given by:
8176 : *
8177 : * log_2 n
8178 : * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6)
8179 : * k = 0
8180 : *
8181 : * And you'll find that:
8182 : *
8183 : * A^(log_2 n)_i,j != 0 for all i,j (7)
8184 : *
8185 : * Showing there's indeed a path between every CPU in at most O(log n) steps.
8186 : * The task movement gives a factor of O(m), giving a convergence complexity
8187 : * of:
8188 : *
8189 : * O(nm log n), n := nr_cpus, m := nr_tasks (8)
8190 : *
8191 : *
8192 : * WORK CONSERVING
8193 : *
8194 : * In order to avoid CPUs going idle while there's still work to do, new idle
8195 : * balancing is more aggressive and has the newly idle CPU iterate up the domain
8196 : * tree itself instead of relying on other CPUs to bring it work.
8197 : *
8198 : * This adds some complexity to both (5) and (8) but it reduces the total idle
8199 : * time.
8200 : *
8201 : * [XXX more?]
8202 : *
8203 : *
8204 : * CGROUPS
8205 : *
8206 : * Cgroups make a horror show out of (2), instead of a simple sum we get:
8207 : *
8208 : * s_k,i
8209 : * W_i,0 = \Sum_j \Prod_k w_k * ----- (9)
8210 : * S_k
8211 : *
8212 : * Where
8213 : *
8214 : * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10)
8215 : *
8216 : * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i.
8217 : *
8218 : * The big problem is S_k, its a global sum needed to compute a local (W_i)
8219 : * property.
8220 : *
8221 : * [XXX write more on how we solve this.. _after_ merging pjt's patches that
8222 : * rewrite all of this once again.]
8223 : */
8224 :
8225 : static unsigned long __read_mostly max_load_balance_interval = HZ/10;
8226 :
8227 : enum fbq_type { regular, remote, all };
8228 :
8229 : /*
8230 : * 'group_type' describes the group of CPUs at the moment of load balancing.
8231 : *
8232 : * The enum is ordered by pulling priority, with the group with lowest priority
8233 : * first so the group_type can simply be compared when selecting the busiest
8234 : * group. See update_sd_pick_busiest().
8235 : */
8236 : enum group_type {
8237 : /* The group has spare capacity that can be used to run more tasks. */
8238 : group_has_spare = 0,
8239 : /*
8240 : * The group is fully used and the tasks don't compete for more CPU
8241 : * cycles. Nevertheless, some tasks might wait before running.
8242 : */
8243 : group_fully_busy,
8244 : /*
8245 : * One task doesn't fit with CPU's capacity and must be migrated to a
8246 : * more powerful CPU.
8247 : */
8248 : group_misfit_task,
8249 : /*
8250 : * SD_ASYM_PACKING only: One local CPU with higher capacity is available,
8251 : * and the task should be migrated to it instead of running on the
8252 : * current CPU.
8253 : */
8254 : group_asym_packing,
8255 : /*
8256 : * The tasks' affinity constraints previously prevented the scheduler
8257 : * from balancing the load across the system.
8258 : */
8259 : group_imbalanced,
8260 : /*
8261 : * The CPU is overloaded and can't provide expected CPU cycles to all
8262 : * tasks.
8263 : */
8264 : group_overloaded
8265 : };
8266 :
8267 : enum migration_type {
8268 : migrate_load = 0,
8269 : migrate_util,
8270 : migrate_task,
8271 : migrate_misfit
8272 : };
8273 :
8274 : #define LBF_ALL_PINNED 0x01
8275 : #define LBF_NEED_BREAK 0x02
8276 : #define LBF_DST_PINNED 0x04
8277 : #define LBF_SOME_PINNED 0x08
8278 : #define LBF_ACTIVE_LB 0x10
8279 :
8280 : struct lb_env {
8281 : struct sched_domain *sd;
8282 :
8283 : struct rq *src_rq;
8284 : int src_cpu;
8285 :
8286 : int dst_cpu;
8287 : struct rq *dst_rq;
8288 :
8289 : struct cpumask *dst_grpmask;
8290 : int new_dst_cpu;
8291 : enum cpu_idle_type idle;
8292 : long imbalance;
8293 : /* The set of CPUs under consideration for load-balancing */
8294 : struct cpumask *cpus;
8295 :
8296 : unsigned int flags;
8297 :
8298 : unsigned int loop;
8299 : unsigned int loop_break;
8300 : unsigned int loop_max;
8301 :
8302 : enum fbq_type fbq_type;
8303 : enum migration_type migration_type;
8304 : struct list_head tasks;
8305 : };
8306 :
8307 : /*
8308 : * Is this task likely cache-hot:
8309 : */
8310 : static int task_hot(struct task_struct *p, struct lb_env *env)
8311 : {
8312 : s64 delta;
8313 :
8314 : lockdep_assert_rq_held(env->src_rq);
8315 :
8316 : if (p->sched_class != &fair_sched_class)
8317 : return 0;
8318 :
8319 : if (unlikely(task_has_idle_policy(p)))
8320 : return 0;
8321 :
8322 : /* SMT siblings share cache */
8323 : if (env->sd->flags & SD_SHARE_CPUCAPACITY)
8324 : return 0;
8325 :
8326 : /*
8327 : * Buddy candidates are cache hot:
8328 : */
8329 : if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
8330 : (&p->se == cfs_rq_of(&p->se)->next ||
8331 : &p->se == cfs_rq_of(&p->se)->last))
8332 : return 1;
8333 :
8334 : if (sysctl_sched_migration_cost == -1)
8335 : return 1;
8336 :
8337 : /*
8338 : * Don't migrate task if the task's cookie does not match
8339 : * with the destination CPU's core cookie.
8340 : */
8341 : if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))
8342 : return 1;
8343 :
8344 : if (sysctl_sched_migration_cost == 0)
8345 : return 0;
8346 :
8347 : delta = rq_clock_task(env->src_rq) - p->se.exec_start;
8348 :
8349 : return delta < (s64)sysctl_sched_migration_cost;
8350 : }
8351 :
8352 : #ifdef CONFIG_NUMA_BALANCING
8353 : /*
8354 : * Returns 1, if task migration degrades locality
8355 : * Returns 0, if task migration improves locality i.e migration preferred.
8356 : * Returns -1, if task migration is not affected by locality.
8357 : */
8358 : static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
8359 : {
8360 : struct numa_group *numa_group = rcu_dereference(p->numa_group);
8361 : unsigned long src_weight, dst_weight;
8362 : int src_nid, dst_nid, dist;
8363 :
8364 : if (!static_branch_likely(&sched_numa_balancing))
8365 : return -1;
8366 :
8367 : if (!p->numa_faults || !(env->sd->flags & SD_NUMA))
8368 : return -1;
8369 :
8370 : src_nid = cpu_to_node(env->src_cpu);
8371 : dst_nid = cpu_to_node(env->dst_cpu);
8372 :
8373 : if (src_nid == dst_nid)
8374 : return -1;
8375 :
8376 : /* Migrating away from the preferred node is always bad. */
8377 : if (src_nid == p->numa_preferred_nid) {
8378 : if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
8379 : return 1;
8380 : else
8381 : return -1;
8382 : }
8383 :
8384 : /* Encourage migration to the preferred node. */
8385 : if (dst_nid == p->numa_preferred_nid)
8386 : return 0;
8387 :
8388 : /* Leaving a core idle is often worse than degrading locality. */
8389 : if (env->idle == CPU_IDLE)
8390 : return -1;
8391 :
8392 : dist = node_distance(src_nid, dst_nid);
8393 : if (numa_group) {
8394 : src_weight = group_weight(p, src_nid, dist);
8395 : dst_weight = group_weight(p, dst_nid, dist);
8396 : } else {
8397 : src_weight = task_weight(p, src_nid, dist);
8398 : dst_weight = task_weight(p, dst_nid, dist);
8399 : }
8400 :
8401 : return dst_weight < src_weight;
8402 : }
8403 :
8404 : #else
8405 : static inline int migrate_degrades_locality(struct task_struct *p,
8406 : struct lb_env *env)
8407 : {
8408 : return -1;
8409 : }
8410 : #endif
8411 :
8412 : /*
8413 : * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
8414 : */
8415 : static
8416 : int can_migrate_task(struct task_struct *p, struct lb_env *env)
8417 : {
8418 : int tsk_cache_hot;
8419 :
8420 : lockdep_assert_rq_held(env->src_rq);
8421 :
8422 : /*
8423 : * We do not migrate tasks that are:
8424 : * 1) throttled_lb_pair, or
8425 : * 2) cannot be migrated to this CPU due to cpus_ptr, or
8426 : * 3) running (obviously), or
8427 : * 4) are cache-hot on their current CPU.
8428 : */
8429 : if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
8430 : return 0;
8431 :
8432 : /* Disregard pcpu kthreads; they are where they need to be. */
8433 : if (kthread_is_per_cpu(p))
8434 : return 0;
8435 :
8436 : if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {
8437 : int cpu;
8438 :
8439 : schedstat_inc(p->stats.nr_failed_migrations_affine);
8440 :
8441 : env->flags |= LBF_SOME_PINNED;
8442 :
8443 : /*
8444 : * Remember if this task can be migrated to any other CPU in
8445 : * our sched_group. We may want to revisit it if we couldn't
8446 : * meet load balance goals by pulling other tasks on src_cpu.
8447 : *
8448 : * Avoid computing new_dst_cpu
8449 : * - for NEWLY_IDLE
8450 : * - if we have already computed one in current iteration
8451 : * - if it's an active balance
8452 : */
8453 : if (env->idle == CPU_NEWLY_IDLE ||
8454 : env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB))
8455 : return 0;
8456 :
8457 : /* Prevent to re-select dst_cpu via env's CPUs: */
8458 : for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
8459 : if (cpumask_test_cpu(cpu, p->cpus_ptr)) {
8460 : env->flags |= LBF_DST_PINNED;
8461 : env->new_dst_cpu = cpu;
8462 : break;
8463 : }
8464 : }
8465 :
8466 : return 0;
8467 : }
8468 :
8469 : /* Record that we found at least one task that could run on dst_cpu */
8470 : env->flags &= ~LBF_ALL_PINNED;
8471 :
8472 : if (task_on_cpu(env->src_rq, p)) {
8473 : schedstat_inc(p->stats.nr_failed_migrations_running);
8474 : return 0;
8475 : }
8476 :
8477 : /*
8478 : * Aggressive migration if:
8479 : * 1) active balance
8480 : * 2) destination numa is preferred
8481 : * 3) task is cache cold, or
8482 : * 4) too many balance attempts have failed.
8483 : */
8484 : if (env->flags & LBF_ACTIVE_LB)
8485 : return 1;
8486 :
8487 : tsk_cache_hot = migrate_degrades_locality(p, env);
8488 : if (tsk_cache_hot == -1)
8489 : tsk_cache_hot = task_hot(p, env);
8490 :
8491 : if (tsk_cache_hot <= 0 ||
8492 : env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
8493 : if (tsk_cache_hot == 1) {
8494 : schedstat_inc(env->sd->lb_hot_gained[env->idle]);
8495 : schedstat_inc(p->stats.nr_forced_migrations);
8496 : }
8497 : return 1;
8498 : }
8499 :
8500 : schedstat_inc(p->stats.nr_failed_migrations_hot);
8501 : return 0;
8502 : }
8503 :
8504 : /*
8505 : * detach_task() -- detach the task for the migration specified in env
8506 : */
8507 : static void detach_task(struct task_struct *p, struct lb_env *env)
8508 : {
8509 : lockdep_assert_rq_held(env->src_rq);
8510 :
8511 : deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);
8512 : set_task_cpu(p, env->dst_cpu);
8513 : }
8514 :
8515 : /*
8516 : * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as
8517 : * part of active balancing operations within "domain".
8518 : *
8519 : * Returns a task if successful and NULL otherwise.
8520 : */
8521 : static struct task_struct *detach_one_task(struct lb_env *env)
8522 : {
8523 : struct task_struct *p;
8524 :
8525 : lockdep_assert_rq_held(env->src_rq);
8526 :
8527 : list_for_each_entry_reverse(p,
8528 : &env->src_rq->cfs_tasks, se.group_node) {
8529 : if (!can_migrate_task(p, env))
8530 : continue;
8531 :
8532 : detach_task(p, env);
8533 :
8534 : /*
8535 : * Right now, this is only the second place where
8536 : * lb_gained[env->idle] is updated (other is detach_tasks)
8537 : * so we can safely collect stats here rather than
8538 : * inside detach_tasks().
8539 : */
8540 : schedstat_inc(env->sd->lb_gained[env->idle]);
8541 : return p;
8542 : }
8543 : return NULL;
8544 : }
8545 :
8546 : /*
8547 : * detach_tasks() -- tries to detach up to imbalance load/util/tasks from
8548 : * busiest_rq, as part of a balancing operation within domain "sd".
8549 : *
8550 : * Returns number of detached tasks if successful and 0 otherwise.
8551 : */
8552 : static int detach_tasks(struct lb_env *env)
8553 : {
8554 : struct list_head *tasks = &env->src_rq->cfs_tasks;
8555 : unsigned long util, load;
8556 : struct task_struct *p;
8557 : int detached = 0;
8558 :
8559 : lockdep_assert_rq_held(env->src_rq);
8560 :
8561 : /*
8562 : * Source run queue has been emptied by another CPU, clear
8563 : * LBF_ALL_PINNED flag as we will not test any task.
8564 : */
8565 : if (env->src_rq->nr_running <= 1) {
8566 : env->flags &= ~LBF_ALL_PINNED;
8567 : return 0;
8568 : }
8569 :
8570 : if (env->imbalance <= 0)
8571 : return 0;
8572 :
8573 : while (!list_empty(tasks)) {
8574 : /*
8575 : * We don't want to steal all, otherwise we may be treated likewise,
8576 : * which could at worst lead to a livelock crash.
8577 : */
8578 : if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
8579 : break;
8580 :
8581 : env->loop++;
8582 : /*
8583 : * We've more or less seen every task there is, call it quits
8584 : * unless we haven't found any movable task yet.
8585 : */
8586 : if (env->loop > env->loop_max &&
8587 : !(env->flags & LBF_ALL_PINNED))
8588 : break;
8589 :
8590 : /* take a breather every nr_migrate tasks */
8591 : if (env->loop > env->loop_break) {
8592 : env->loop_break += SCHED_NR_MIGRATE_BREAK;
8593 : env->flags |= LBF_NEED_BREAK;
8594 : break;
8595 : }
8596 :
8597 : p = list_last_entry(tasks, struct task_struct, se.group_node);
8598 :
8599 : if (!can_migrate_task(p, env))
8600 : goto next;
8601 :
8602 : switch (env->migration_type) {
8603 : case migrate_load:
8604 : /*
8605 : * Depending of the number of CPUs and tasks and the
8606 : * cgroup hierarchy, task_h_load() can return a null
8607 : * value. Make sure that env->imbalance decreases
8608 : * otherwise detach_tasks() will stop only after
8609 : * detaching up to loop_max tasks.
8610 : */
8611 : load = max_t(unsigned long, task_h_load(p), 1);
8612 :
8613 : if (sched_feat(LB_MIN) &&
8614 : load < 16 && !env->sd->nr_balance_failed)
8615 : goto next;
8616 :
8617 : /*
8618 : * Make sure that we don't migrate too much load.
8619 : * Nevertheless, let relax the constraint if
8620 : * scheduler fails to find a good waiting task to
8621 : * migrate.
8622 : */
8623 : if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)
8624 : goto next;
8625 :
8626 : env->imbalance -= load;
8627 : break;
8628 :
8629 : case migrate_util:
8630 : util = task_util_est(p);
8631 :
8632 : if (util > env->imbalance)
8633 : goto next;
8634 :
8635 : env->imbalance -= util;
8636 : break;
8637 :
8638 : case migrate_task:
8639 : env->imbalance--;
8640 : break;
8641 :
8642 : case migrate_misfit:
8643 : /* This is not a misfit task */
8644 : if (task_fits_cpu(p, env->src_cpu))
8645 : goto next;
8646 :
8647 : env->imbalance = 0;
8648 : break;
8649 : }
8650 :
8651 : detach_task(p, env);
8652 : list_add(&p->se.group_node, &env->tasks);
8653 :
8654 : detached++;
8655 :
8656 : #ifdef CONFIG_PREEMPTION
8657 : /*
8658 : * NEWIDLE balancing is a source of latency, so preemptible
8659 : * kernels will stop after the first task is detached to minimize
8660 : * the critical section.
8661 : */
8662 : if (env->idle == CPU_NEWLY_IDLE)
8663 : break;
8664 : #endif
8665 :
8666 : /*
8667 : * We only want to steal up to the prescribed amount of
8668 : * load/util/tasks.
8669 : */
8670 : if (env->imbalance <= 0)
8671 : break;
8672 :
8673 : continue;
8674 : next:
8675 : list_move(&p->se.group_node, tasks);
8676 : }
8677 :
8678 : /*
8679 : * Right now, this is one of only two places we collect this stat
8680 : * so we can safely collect detach_one_task() stats here rather
8681 : * than inside detach_one_task().
8682 : */
8683 : schedstat_add(env->sd->lb_gained[env->idle], detached);
8684 :
8685 : return detached;
8686 : }
8687 :
8688 : /*
8689 : * attach_task() -- attach the task detached by detach_task() to its new rq.
8690 : */
8691 : static void attach_task(struct rq *rq, struct task_struct *p)
8692 : {
8693 : lockdep_assert_rq_held(rq);
8694 :
8695 : WARN_ON_ONCE(task_rq(p) != rq);
8696 : activate_task(rq, p, ENQUEUE_NOCLOCK);
8697 : check_preempt_curr(rq, p, 0);
8698 : }
8699 :
8700 : /*
8701 : * attach_one_task() -- attaches the task returned from detach_one_task() to
8702 : * its new rq.
8703 : */
8704 : static void attach_one_task(struct rq *rq, struct task_struct *p)
8705 : {
8706 : struct rq_flags rf;
8707 :
8708 : rq_lock(rq, &rf);
8709 : update_rq_clock(rq);
8710 : attach_task(rq, p);
8711 : rq_unlock(rq, &rf);
8712 : }
8713 :
8714 : /*
8715 : * attach_tasks() -- attaches all tasks detached by detach_tasks() to their
8716 : * new rq.
8717 : */
8718 : static void attach_tasks(struct lb_env *env)
8719 : {
8720 : struct list_head *tasks = &env->tasks;
8721 : struct task_struct *p;
8722 : struct rq_flags rf;
8723 :
8724 : rq_lock(env->dst_rq, &rf);
8725 : update_rq_clock(env->dst_rq);
8726 :
8727 : while (!list_empty(tasks)) {
8728 : p = list_first_entry(tasks, struct task_struct, se.group_node);
8729 : list_del_init(&p->se.group_node);
8730 :
8731 : attach_task(env->dst_rq, p);
8732 : }
8733 :
8734 : rq_unlock(env->dst_rq, &rf);
8735 : }
8736 :
8737 : #ifdef CONFIG_NO_HZ_COMMON
8738 : static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq)
8739 : {
8740 : if (cfs_rq->avg.load_avg)
8741 : return true;
8742 :
8743 : if (cfs_rq->avg.util_avg)
8744 : return true;
8745 :
8746 : return false;
8747 : }
8748 :
8749 : static inline bool others_have_blocked(struct rq *rq)
8750 : {
8751 : if (READ_ONCE(rq->avg_rt.util_avg))
8752 : return true;
8753 :
8754 : if (READ_ONCE(rq->avg_dl.util_avg))
8755 : return true;
8756 :
8757 : if (thermal_load_avg(rq))
8758 : return true;
8759 :
8760 : #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
8761 : if (READ_ONCE(rq->avg_irq.util_avg))
8762 : return true;
8763 : #endif
8764 :
8765 : return false;
8766 : }
8767 :
8768 : static inline void update_blocked_load_tick(struct rq *rq)
8769 : {
8770 : WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies);
8771 : }
8772 :
8773 : static inline void update_blocked_load_status(struct rq *rq, bool has_blocked)
8774 : {
8775 : if (!has_blocked)
8776 : rq->has_blocked_load = 0;
8777 : }
8778 : #else
8779 : static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; }
8780 : static inline bool others_have_blocked(struct rq *rq) { return false; }
8781 : static inline void update_blocked_load_tick(struct rq *rq) {}
8782 : static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {}
8783 : #endif
8784 :
8785 : static bool __update_blocked_others(struct rq *rq, bool *done)
8786 : {
8787 : const struct sched_class *curr_class;
8788 : u64 now = rq_clock_pelt(rq);
8789 : unsigned long thermal_pressure;
8790 : bool decayed;
8791 :
8792 : /*
8793 : * update_load_avg() can call cpufreq_update_util(). Make sure that RT,
8794 : * DL and IRQ signals have been updated before updating CFS.
8795 : */
8796 : curr_class = rq->curr->sched_class;
8797 :
8798 : thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq));
8799 :
8800 : decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) |
8801 : update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) |
8802 : update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) |
8803 : update_irq_load_avg(rq, 0);
8804 :
8805 : if (others_have_blocked(rq))
8806 : *done = false;
8807 :
8808 : return decayed;
8809 : }
8810 :
8811 : #ifdef CONFIG_FAIR_GROUP_SCHED
8812 :
8813 : static bool __update_blocked_fair(struct rq *rq, bool *done)
8814 : {
8815 : struct cfs_rq *cfs_rq, *pos;
8816 : bool decayed = false;
8817 : int cpu = cpu_of(rq);
8818 :
8819 : /*
8820 : * Iterates the task_group tree in a bottom up fashion, see
8821 : * list_add_leaf_cfs_rq() for details.
8822 : */
8823 : for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) {
8824 : struct sched_entity *se;
8825 :
8826 : if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) {
8827 : update_tg_load_avg(cfs_rq);
8828 :
8829 : if (cfs_rq->nr_running == 0)
8830 : update_idle_cfs_rq_clock_pelt(cfs_rq);
8831 :
8832 : if (cfs_rq == &rq->cfs)
8833 : decayed = true;
8834 : }
8835 :
8836 : /* Propagate pending load changes to the parent, if any: */
8837 : se = cfs_rq->tg->se[cpu];
8838 : if (se && !skip_blocked_update(se))
8839 : update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
8840 :
8841 : /*
8842 : * There can be a lot of idle CPU cgroups. Don't let fully
8843 : * decayed cfs_rqs linger on the list.
8844 : */
8845 : if (cfs_rq_is_decayed(cfs_rq))
8846 : list_del_leaf_cfs_rq(cfs_rq);
8847 :
8848 : /* Don't need periodic decay once load/util_avg are null */
8849 : if (cfs_rq_has_blocked(cfs_rq))
8850 : *done = false;
8851 : }
8852 :
8853 : return decayed;
8854 : }
8855 :
8856 : /*
8857 : * Compute the hierarchical load factor for cfs_rq and all its ascendants.
8858 : * This needs to be done in a top-down fashion because the load of a child
8859 : * group is a fraction of its parents load.
8860 : */
8861 : static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
8862 : {
8863 : struct rq *rq = rq_of(cfs_rq);
8864 : struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
8865 : unsigned long now = jiffies;
8866 : unsigned long load;
8867 :
8868 : if (cfs_rq->last_h_load_update == now)
8869 : return;
8870 :
8871 : WRITE_ONCE(cfs_rq->h_load_next, NULL);
8872 : for_each_sched_entity(se) {
8873 : cfs_rq = cfs_rq_of(se);
8874 : WRITE_ONCE(cfs_rq->h_load_next, se);
8875 : if (cfs_rq->last_h_load_update == now)
8876 : break;
8877 : }
8878 :
8879 : if (!se) {
8880 : cfs_rq->h_load = cfs_rq_load_avg(cfs_rq);
8881 : cfs_rq->last_h_load_update = now;
8882 : }
8883 :
8884 : while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) {
8885 : load = cfs_rq->h_load;
8886 : load = div64_ul(load * se->avg.load_avg,
8887 : cfs_rq_load_avg(cfs_rq) + 1);
8888 : cfs_rq = group_cfs_rq(se);
8889 : cfs_rq->h_load = load;
8890 : cfs_rq->last_h_load_update = now;
8891 : }
8892 : }
8893 :
8894 : static unsigned long task_h_load(struct task_struct *p)
8895 : {
8896 : struct cfs_rq *cfs_rq = task_cfs_rq(p);
8897 :
8898 : update_cfs_rq_h_load(cfs_rq);
8899 : return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,
8900 : cfs_rq_load_avg(cfs_rq) + 1);
8901 : }
8902 : #else
8903 : static bool __update_blocked_fair(struct rq *rq, bool *done)
8904 : {
8905 : struct cfs_rq *cfs_rq = &rq->cfs;
8906 : bool decayed;
8907 :
8908 : decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq);
8909 : if (cfs_rq_has_blocked(cfs_rq))
8910 : *done = false;
8911 :
8912 : return decayed;
8913 : }
8914 :
8915 : static unsigned long task_h_load(struct task_struct *p)
8916 : {
8917 : return p->se.avg.load_avg;
8918 : }
8919 : #endif
8920 :
8921 : static void update_blocked_averages(int cpu)
8922 : {
8923 : bool decayed = false, done = true;
8924 : struct rq *rq = cpu_rq(cpu);
8925 : struct rq_flags rf;
8926 :
8927 : rq_lock_irqsave(rq, &rf);
8928 : update_blocked_load_tick(rq);
8929 : update_rq_clock(rq);
8930 :
8931 : decayed |= __update_blocked_others(rq, &done);
8932 : decayed |= __update_blocked_fair(rq, &done);
8933 :
8934 : update_blocked_load_status(rq, !done);
8935 : if (decayed)
8936 : cpufreq_update_util(rq, 0);
8937 : rq_unlock_irqrestore(rq, &rf);
8938 : }
8939 :
8940 : /********** Helpers for find_busiest_group ************************/
8941 :
8942 : /*
8943 : * sg_lb_stats - stats of a sched_group required for load_balancing
8944 : */
8945 : struct sg_lb_stats {
8946 : unsigned long avg_load; /*Avg load across the CPUs of the group */
8947 : unsigned long group_load; /* Total load over the CPUs of the group */
8948 : unsigned long group_capacity;
8949 : unsigned long group_util; /* Total utilization over the CPUs of the group */
8950 : unsigned long group_runnable; /* Total runnable time over the CPUs of the group */
8951 : unsigned int sum_nr_running; /* Nr of tasks running in the group */
8952 : unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */
8953 : unsigned int idle_cpus;
8954 : unsigned int group_weight;
8955 : enum group_type group_type;
8956 : unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */
8957 : unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */
8958 : #ifdef CONFIG_NUMA_BALANCING
8959 : unsigned int nr_numa_running;
8960 : unsigned int nr_preferred_running;
8961 : #endif
8962 : };
8963 :
8964 : /*
8965 : * sd_lb_stats - Structure to store the statistics of a sched_domain
8966 : * during load balancing.
8967 : */
8968 : struct sd_lb_stats {
8969 : struct sched_group *busiest; /* Busiest group in this sd */
8970 : struct sched_group *local; /* Local group in this sd */
8971 : unsigned long total_load; /* Total load of all groups in sd */
8972 : unsigned long total_capacity; /* Total capacity of all groups in sd */
8973 : unsigned long avg_load; /* Average load across all groups in sd */
8974 : unsigned int prefer_sibling; /* tasks should go to sibling first */
8975 :
8976 : struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
8977 : struct sg_lb_stats local_stat; /* Statistics of the local group */
8978 : };
8979 :
8980 : static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
8981 : {
8982 : /*
8983 : * Skimp on the clearing to avoid duplicate work. We can avoid clearing
8984 : * local_stat because update_sg_lb_stats() does a full clear/assignment.
8985 : * We must however set busiest_stat::group_type and
8986 : * busiest_stat::idle_cpus to the worst busiest group because
8987 : * update_sd_pick_busiest() reads these before assignment.
8988 : */
8989 : *sds = (struct sd_lb_stats){
8990 : .busiest = NULL,
8991 : .local = NULL,
8992 : .total_load = 0UL,
8993 : .total_capacity = 0UL,
8994 : .busiest_stat = {
8995 : .idle_cpus = UINT_MAX,
8996 : .group_type = group_has_spare,
8997 : },
8998 : };
8999 : }
9000 :
9001 : static unsigned long scale_rt_capacity(int cpu)
9002 : {
9003 : struct rq *rq = cpu_rq(cpu);
9004 : unsigned long max = arch_scale_cpu_capacity(cpu);
9005 : unsigned long used, free;
9006 : unsigned long irq;
9007 :
9008 : irq = cpu_util_irq(rq);
9009 :
9010 : if (unlikely(irq >= max))
9011 : return 1;
9012 :
9013 : /*
9014 : * avg_rt.util_avg and avg_dl.util_avg track binary signals
9015 : * (running and not running) with weights 0 and 1024 respectively.
9016 : * avg_thermal.load_avg tracks thermal pressure and the weighted
9017 : * average uses the actual delta max capacity(load).
9018 : */
9019 : used = READ_ONCE(rq->avg_rt.util_avg);
9020 : used += READ_ONCE(rq->avg_dl.util_avg);
9021 : used += thermal_load_avg(rq);
9022 :
9023 : if (unlikely(used >= max))
9024 : return 1;
9025 :
9026 : free = max - used;
9027 :
9028 : return scale_irq_capacity(free, irq, max);
9029 : }
9030 :
9031 : static void update_cpu_capacity(struct sched_domain *sd, int cpu)
9032 : {
9033 : unsigned long capacity = scale_rt_capacity(cpu);
9034 : struct sched_group *sdg = sd->groups;
9035 :
9036 : cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu);
9037 :
9038 : if (!capacity)
9039 : capacity = 1;
9040 :
9041 : cpu_rq(cpu)->cpu_capacity = capacity;
9042 : trace_sched_cpu_capacity_tp(cpu_rq(cpu));
9043 :
9044 : sdg->sgc->capacity = capacity;
9045 : sdg->sgc->min_capacity = capacity;
9046 : sdg->sgc->max_capacity = capacity;
9047 : }
9048 :
9049 : void update_group_capacity(struct sched_domain *sd, int cpu)
9050 : {
9051 : struct sched_domain *child = sd->child;
9052 : struct sched_group *group, *sdg = sd->groups;
9053 : unsigned long capacity, min_capacity, max_capacity;
9054 : unsigned long interval;
9055 :
9056 : interval = msecs_to_jiffies(sd->balance_interval);
9057 : interval = clamp(interval, 1UL, max_load_balance_interval);
9058 : sdg->sgc->next_update = jiffies + interval;
9059 :
9060 : if (!child) {
9061 : update_cpu_capacity(sd, cpu);
9062 : return;
9063 : }
9064 :
9065 : capacity = 0;
9066 : min_capacity = ULONG_MAX;
9067 : max_capacity = 0;
9068 :
9069 : if (child->flags & SD_OVERLAP) {
9070 : /*
9071 : * SD_OVERLAP domains cannot assume that child groups
9072 : * span the current group.
9073 : */
9074 :
9075 : for_each_cpu(cpu, sched_group_span(sdg)) {
9076 : unsigned long cpu_cap = capacity_of(cpu);
9077 :
9078 : capacity += cpu_cap;
9079 : min_capacity = min(cpu_cap, min_capacity);
9080 : max_capacity = max(cpu_cap, max_capacity);
9081 : }
9082 : } else {
9083 : /*
9084 : * !SD_OVERLAP domains can assume that child groups
9085 : * span the current group.
9086 : */
9087 :
9088 : group = child->groups;
9089 : do {
9090 : struct sched_group_capacity *sgc = group->sgc;
9091 :
9092 : capacity += sgc->capacity;
9093 : min_capacity = min(sgc->min_capacity, min_capacity);
9094 : max_capacity = max(sgc->max_capacity, max_capacity);
9095 : group = group->next;
9096 : } while (group != child->groups);
9097 : }
9098 :
9099 : sdg->sgc->capacity = capacity;
9100 : sdg->sgc->min_capacity = min_capacity;
9101 : sdg->sgc->max_capacity = max_capacity;
9102 : }
9103 :
9104 : /*
9105 : * Check whether the capacity of the rq has been noticeably reduced by side
9106 : * activity. The imbalance_pct is used for the threshold.
9107 : * Return true is the capacity is reduced
9108 : */
9109 : static inline int
9110 : check_cpu_capacity(struct rq *rq, struct sched_domain *sd)
9111 : {
9112 : return ((rq->cpu_capacity * sd->imbalance_pct) <
9113 : (rq->cpu_capacity_orig * 100));
9114 : }
9115 :
9116 : /*
9117 : * Check whether a rq has a misfit task and if it looks like we can actually
9118 : * help that task: we can migrate the task to a CPU of higher capacity, or
9119 : * the task's current CPU is heavily pressured.
9120 : */
9121 : static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd)
9122 : {
9123 : return rq->misfit_task_load &&
9124 : (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity ||
9125 : check_cpu_capacity(rq, sd));
9126 : }
9127 :
9128 : /*
9129 : * Group imbalance indicates (and tries to solve) the problem where balancing
9130 : * groups is inadequate due to ->cpus_ptr constraints.
9131 : *
9132 : * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a
9133 : * cpumask covering 1 CPU of the first group and 3 CPUs of the second group.
9134 : * Something like:
9135 : *
9136 : * { 0 1 2 3 } { 4 5 6 7 }
9137 : * * * * *
9138 : *
9139 : * If we were to balance group-wise we'd place two tasks in the first group and
9140 : * two tasks in the second group. Clearly this is undesired as it will overload
9141 : * cpu 3 and leave one of the CPUs in the second group unused.
9142 : *
9143 : * The current solution to this issue is detecting the skew in the first group
9144 : * by noticing the lower domain failed to reach balance and had difficulty
9145 : * moving tasks due to affinity constraints.
9146 : *
9147 : * When this is so detected; this group becomes a candidate for busiest; see
9148 : * update_sd_pick_busiest(). And calculate_imbalance() and
9149 : * find_busiest_group() avoid some of the usual balance conditions to allow it
9150 : * to create an effective group imbalance.
9151 : *
9152 : * This is a somewhat tricky proposition since the next run might not find the
9153 : * group imbalance and decide the groups need to be balanced again. A most
9154 : * subtle and fragile situation.
9155 : */
9156 :
9157 : static inline int sg_imbalanced(struct sched_group *group)
9158 : {
9159 : return group->sgc->imbalance;
9160 : }
9161 :
9162 : /*
9163 : * group_has_capacity returns true if the group has spare capacity that could
9164 : * be used by some tasks.
9165 : * We consider that a group has spare capacity if the number of task is
9166 : * smaller than the number of CPUs or if the utilization is lower than the
9167 : * available capacity for CFS tasks.
9168 : * For the latter, we use a threshold to stabilize the state, to take into
9169 : * account the variance of the tasks' load and to return true if the available
9170 : * capacity in meaningful for the load balancer.
9171 : * As an example, an available capacity of 1% can appear but it doesn't make
9172 : * any benefit for the load balance.
9173 : */
9174 : static inline bool
9175 : group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
9176 : {
9177 : if (sgs->sum_nr_running < sgs->group_weight)
9178 : return true;
9179 :
9180 : if ((sgs->group_capacity * imbalance_pct) <
9181 : (sgs->group_runnable * 100))
9182 : return false;
9183 :
9184 : if ((sgs->group_capacity * 100) >
9185 : (sgs->group_util * imbalance_pct))
9186 : return true;
9187 :
9188 : return false;
9189 : }
9190 :
9191 : /*
9192 : * group_is_overloaded returns true if the group has more tasks than it can
9193 : * handle.
9194 : * group_is_overloaded is not equals to !group_has_capacity because a group
9195 : * with the exact right number of tasks, has no more spare capacity but is not
9196 : * overloaded so both group_has_capacity and group_is_overloaded return
9197 : * false.
9198 : */
9199 : static inline bool
9200 : group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
9201 : {
9202 : if (sgs->sum_nr_running <= sgs->group_weight)
9203 : return false;
9204 :
9205 : if ((sgs->group_capacity * 100) <
9206 : (sgs->group_util * imbalance_pct))
9207 : return true;
9208 :
9209 : if ((sgs->group_capacity * imbalance_pct) <
9210 : (sgs->group_runnable * 100))
9211 : return true;
9212 :
9213 : return false;
9214 : }
9215 :
9216 : static inline enum
9217 : group_type group_classify(unsigned int imbalance_pct,
9218 : struct sched_group *group,
9219 : struct sg_lb_stats *sgs)
9220 : {
9221 : if (group_is_overloaded(imbalance_pct, sgs))
9222 : return group_overloaded;
9223 :
9224 : if (sg_imbalanced(group))
9225 : return group_imbalanced;
9226 :
9227 : if (sgs->group_asym_packing)
9228 : return group_asym_packing;
9229 :
9230 : if (sgs->group_misfit_task_load)
9231 : return group_misfit_task;
9232 :
9233 : if (!group_has_capacity(imbalance_pct, sgs))
9234 : return group_fully_busy;
9235 :
9236 : return group_has_spare;
9237 : }
9238 :
9239 : /**
9240 : * asym_smt_can_pull_tasks - Check whether the load balancing CPU can pull tasks
9241 : * @dst_cpu: Destination CPU of the load balancing
9242 : * @sds: Load-balancing data with statistics of the local group
9243 : * @sgs: Load-balancing statistics of the candidate busiest group
9244 : * @sg: The candidate busiest group
9245 : *
9246 : * Check the state of the SMT siblings of both @sds::local and @sg and decide
9247 : * if @dst_cpu can pull tasks.
9248 : *
9249 : * If @dst_cpu does not have SMT siblings, it can pull tasks if two or more of
9250 : * the SMT siblings of @sg are busy. If only one CPU in @sg is busy, pull tasks
9251 : * only if @dst_cpu has higher priority.
9252 : *
9253 : * If both @dst_cpu and @sg have SMT siblings, and @sg has exactly one more
9254 : * busy CPU than @sds::local, let @dst_cpu pull tasks if it has higher priority.
9255 : * Bigger imbalances in the number of busy CPUs will be dealt with in
9256 : * update_sd_pick_busiest().
9257 : *
9258 : * If @sg does not have SMT siblings, only pull tasks if all of the SMT siblings
9259 : * of @dst_cpu are idle and @sg has lower priority.
9260 : *
9261 : * Return: true if @dst_cpu can pull tasks, false otherwise.
9262 : */
9263 : static bool asym_smt_can_pull_tasks(int dst_cpu, struct sd_lb_stats *sds,
9264 : struct sg_lb_stats *sgs,
9265 : struct sched_group *sg)
9266 : {
9267 : #ifdef CONFIG_SCHED_SMT
9268 : bool local_is_smt, sg_is_smt;
9269 : int sg_busy_cpus;
9270 :
9271 : local_is_smt = sds->local->flags & SD_SHARE_CPUCAPACITY;
9272 : sg_is_smt = sg->flags & SD_SHARE_CPUCAPACITY;
9273 :
9274 : sg_busy_cpus = sgs->group_weight - sgs->idle_cpus;
9275 :
9276 : if (!local_is_smt) {
9277 : /*
9278 : * If we are here, @dst_cpu is idle and does not have SMT
9279 : * siblings. Pull tasks if candidate group has two or more
9280 : * busy CPUs.
9281 : */
9282 : if (sg_busy_cpus >= 2) /* implies sg_is_smt */
9283 : return true;
9284 :
9285 : /*
9286 : * @dst_cpu does not have SMT siblings. @sg may have SMT
9287 : * siblings and only one is busy. In such case, @dst_cpu
9288 : * can help if it has higher priority and is idle (i.e.,
9289 : * it has no running tasks).
9290 : */
9291 : return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);
9292 : }
9293 :
9294 : /* @dst_cpu has SMT siblings. */
9295 :
9296 : if (sg_is_smt) {
9297 : int local_busy_cpus = sds->local->group_weight -
9298 : sds->local_stat.idle_cpus;
9299 : int busy_cpus_delta = sg_busy_cpus - local_busy_cpus;
9300 :
9301 : if (busy_cpus_delta == 1)
9302 : return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);
9303 :
9304 : return false;
9305 : }
9306 :
9307 : /*
9308 : * @sg does not have SMT siblings. Ensure that @sds::local does not end
9309 : * up with more than one busy SMT sibling and only pull tasks if there
9310 : * are not busy CPUs (i.e., no CPU has running tasks).
9311 : */
9312 : if (!sds->local_stat.sum_nr_running)
9313 : return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu);
9314 :
9315 : return false;
9316 : #else
9317 : /* Always return false so that callers deal with non-SMT cases. */
9318 : return false;
9319 : #endif
9320 : }
9321 :
9322 : static inline bool
9323 : sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs,
9324 : struct sched_group *group)
9325 : {
9326 : /* Only do SMT checks if either local or candidate have SMT siblings */
9327 : if ((sds->local->flags & SD_SHARE_CPUCAPACITY) ||
9328 : (group->flags & SD_SHARE_CPUCAPACITY))
9329 : return asym_smt_can_pull_tasks(env->dst_cpu, sds, sgs, group);
9330 :
9331 : return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu);
9332 : }
9333 :
9334 : static inline bool
9335 : sched_reduced_capacity(struct rq *rq, struct sched_domain *sd)
9336 : {
9337 : /*
9338 : * When there is more than 1 task, the group_overloaded case already
9339 : * takes care of cpu with reduced capacity
9340 : */
9341 : if (rq->cfs.h_nr_running != 1)
9342 : return false;
9343 :
9344 : return check_cpu_capacity(rq, sd);
9345 : }
9346 :
9347 : /**
9348 : * update_sg_lb_stats - Update sched_group's statistics for load balancing.
9349 : * @env: The load balancing environment.
9350 : * @sds: Load-balancing data with statistics of the local group.
9351 : * @group: sched_group whose statistics are to be updated.
9352 : * @sgs: variable to hold the statistics for this group.
9353 : * @sg_status: Holds flag indicating the status of the sched_group
9354 : */
9355 : static inline void update_sg_lb_stats(struct lb_env *env,
9356 : struct sd_lb_stats *sds,
9357 : struct sched_group *group,
9358 : struct sg_lb_stats *sgs,
9359 : int *sg_status)
9360 : {
9361 : int i, nr_running, local_group;
9362 :
9363 : memset(sgs, 0, sizeof(*sgs));
9364 :
9365 : local_group = group == sds->local;
9366 :
9367 : for_each_cpu_and(i, sched_group_span(group), env->cpus) {
9368 : struct rq *rq = cpu_rq(i);
9369 : unsigned long load = cpu_load(rq);
9370 :
9371 : sgs->group_load += load;
9372 : sgs->group_util += cpu_util_cfs(i);
9373 : sgs->group_runnable += cpu_runnable(rq);
9374 : sgs->sum_h_nr_running += rq->cfs.h_nr_running;
9375 :
9376 : nr_running = rq->nr_running;
9377 : sgs->sum_nr_running += nr_running;
9378 :
9379 : if (nr_running > 1)
9380 : *sg_status |= SG_OVERLOAD;
9381 :
9382 : if (cpu_overutilized(i))
9383 : *sg_status |= SG_OVERUTILIZED;
9384 :
9385 : #ifdef CONFIG_NUMA_BALANCING
9386 : sgs->nr_numa_running += rq->nr_numa_running;
9387 : sgs->nr_preferred_running += rq->nr_preferred_running;
9388 : #endif
9389 : /*
9390 : * No need to call idle_cpu() if nr_running is not 0
9391 : */
9392 : if (!nr_running && idle_cpu(i)) {
9393 : sgs->idle_cpus++;
9394 : /* Idle cpu can't have misfit task */
9395 : continue;
9396 : }
9397 :
9398 : if (local_group)
9399 : continue;
9400 :
9401 : if (env->sd->flags & SD_ASYM_CPUCAPACITY) {
9402 : /* Check for a misfit task on the cpu */
9403 : if (sgs->group_misfit_task_load < rq->misfit_task_load) {
9404 : sgs->group_misfit_task_load = rq->misfit_task_load;
9405 : *sg_status |= SG_OVERLOAD;
9406 : }
9407 : } else if ((env->idle != CPU_NOT_IDLE) &&
9408 : sched_reduced_capacity(rq, env->sd)) {
9409 : /* Check for a task running on a CPU with reduced capacity */
9410 : if (sgs->group_misfit_task_load < load)
9411 : sgs->group_misfit_task_load = load;
9412 : }
9413 : }
9414 :
9415 : sgs->group_capacity = group->sgc->capacity;
9416 :
9417 : sgs->group_weight = group->group_weight;
9418 :
9419 : /* Check if dst CPU is idle and preferred to this group */
9420 : if (!local_group && env->sd->flags & SD_ASYM_PACKING &&
9421 : env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running &&
9422 : sched_asym(env, sds, sgs, group)) {
9423 : sgs->group_asym_packing = 1;
9424 : }
9425 :
9426 : sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);
9427 :
9428 : /* Computing avg_load makes sense only when group is overloaded */
9429 : if (sgs->group_type == group_overloaded)
9430 : sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
9431 : sgs->group_capacity;
9432 : }
9433 :
9434 : /**
9435 : * update_sd_pick_busiest - return 1 on busiest group
9436 : * @env: The load balancing environment.
9437 : * @sds: sched_domain statistics
9438 : * @sg: sched_group candidate to be checked for being the busiest
9439 : * @sgs: sched_group statistics
9440 : *
9441 : * Determine if @sg is a busier group than the previously selected
9442 : * busiest group.
9443 : *
9444 : * Return: %true if @sg is a busier group than the previously selected
9445 : * busiest group. %false otherwise.
9446 : */
9447 : static bool update_sd_pick_busiest(struct lb_env *env,
9448 : struct sd_lb_stats *sds,
9449 : struct sched_group *sg,
9450 : struct sg_lb_stats *sgs)
9451 : {
9452 : struct sg_lb_stats *busiest = &sds->busiest_stat;
9453 :
9454 : /* Make sure that there is at least one task to pull */
9455 : if (!sgs->sum_h_nr_running)
9456 : return false;
9457 :
9458 : /*
9459 : * Don't try to pull misfit tasks we can't help.
9460 : * We can use max_capacity here as reduction in capacity on some
9461 : * CPUs in the group should either be possible to resolve
9462 : * internally or be covered by avg_load imbalance (eventually).
9463 : */
9464 : if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
9465 : (sgs->group_type == group_misfit_task) &&
9466 : (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
9467 : sds->local_stat.group_type != group_has_spare))
9468 : return false;
9469 :
9470 : if (sgs->group_type > busiest->group_type)
9471 : return true;
9472 :
9473 : if (sgs->group_type < busiest->group_type)
9474 : return false;
9475 :
9476 : /*
9477 : * The candidate and the current busiest group are the same type of
9478 : * group. Let check which one is the busiest according to the type.
9479 : */
9480 :
9481 : switch (sgs->group_type) {
9482 : case group_overloaded:
9483 : /* Select the overloaded group with highest avg_load. */
9484 : if (sgs->avg_load <= busiest->avg_load)
9485 : return false;
9486 : break;
9487 :
9488 : case group_imbalanced:
9489 : /*
9490 : * Select the 1st imbalanced group as we don't have any way to
9491 : * choose one more than another.
9492 : */
9493 : return false;
9494 :
9495 : case group_asym_packing:
9496 : /* Prefer to move from lowest priority CPU's work */
9497 : if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu))
9498 : return false;
9499 : break;
9500 :
9501 : case group_misfit_task:
9502 : /*
9503 : * If we have more than one misfit sg go with the biggest
9504 : * misfit.
9505 : */
9506 : if (sgs->group_misfit_task_load < busiest->group_misfit_task_load)
9507 : return false;
9508 : break;
9509 :
9510 : case group_fully_busy:
9511 : /*
9512 : * Select the fully busy group with highest avg_load. In
9513 : * theory, there is no need to pull task from such kind of
9514 : * group because tasks have all compute capacity that they need
9515 : * but we can still improve the overall throughput by reducing
9516 : * contention when accessing shared HW resources.
9517 : *
9518 : * XXX for now avg_load is not computed and always 0 so we
9519 : * select the 1st one.
9520 : */
9521 : if (sgs->avg_load <= busiest->avg_load)
9522 : return false;
9523 : break;
9524 :
9525 : case group_has_spare:
9526 : /*
9527 : * Select not overloaded group with lowest number of idle cpus
9528 : * and highest number of running tasks. We could also compare
9529 : * the spare capacity which is more stable but it can end up
9530 : * that the group has less spare capacity but finally more idle
9531 : * CPUs which means less opportunity to pull tasks.
9532 : */
9533 : if (sgs->idle_cpus > busiest->idle_cpus)
9534 : return false;
9535 : else if ((sgs->idle_cpus == busiest->idle_cpus) &&
9536 : (sgs->sum_nr_running <= busiest->sum_nr_running))
9537 : return false;
9538 :
9539 : break;
9540 : }
9541 :
9542 : /*
9543 : * Candidate sg has no more than one task per CPU and has higher
9544 : * per-CPU capacity. Migrating tasks to less capable CPUs may harm
9545 : * throughput. Maximize throughput, power/energy consequences are not
9546 : * considered.
9547 : */
9548 : if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
9549 : (sgs->group_type <= group_fully_busy) &&
9550 : (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))
9551 : return false;
9552 :
9553 : return true;
9554 : }
9555 :
9556 : #ifdef CONFIG_NUMA_BALANCING
9557 : static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
9558 : {
9559 : if (sgs->sum_h_nr_running > sgs->nr_numa_running)
9560 : return regular;
9561 : if (sgs->sum_h_nr_running > sgs->nr_preferred_running)
9562 : return remote;
9563 : return all;
9564 : }
9565 :
9566 : static inline enum fbq_type fbq_classify_rq(struct rq *rq)
9567 : {
9568 : if (rq->nr_running > rq->nr_numa_running)
9569 : return regular;
9570 : if (rq->nr_running > rq->nr_preferred_running)
9571 : return remote;
9572 : return all;
9573 : }
9574 : #else
9575 : static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
9576 : {
9577 : return all;
9578 : }
9579 :
9580 : static inline enum fbq_type fbq_classify_rq(struct rq *rq)
9581 : {
9582 : return regular;
9583 : }
9584 : #endif /* CONFIG_NUMA_BALANCING */
9585 :
9586 :
9587 : struct sg_lb_stats;
9588 :
9589 : /*
9590 : * task_running_on_cpu - return 1 if @p is running on @cpu.
9591 : */
9592 :
9593 : static unsigned int task_running_on_cpu(int cpu, struct task_struct *p)
9594 : {
9595 : /* Task has no contribution or is new */
9596 : if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
9597 : return 0;
9598 :
9599 : if (task_on_rq_queued(p))
9600 : return 1;
9601 :
9602 : return 0;
9603 : }
9604 :
9605 : /**
9606 : * idle_cpu_without - would a given CPU be idle without p ?
9607 : * @cpu: the processor on which idleness is tested.
9608 : * @p: task which should be ignored.
9609 : *
9610 : * Return: 1 if the CPU would be idle. 0 otherwise.
9611 : */
9612 : static int idle_cpu_without(int cpu, struct task_struct *p)
9613 : {
9614 : struct rq *rq = cpu_rq(cpu);
9615 :
9616 : if (rq->curr != rq->idle && rq->curr != p)
9617 : return 0;
9618 :
9619 : /*
9620 : * rq->nr_running can't be used but an updated version without the
9621 : * impact of p on cpu must be used instead. The updated nr_running
9622 : * be computed and tested before calling idle_cpu_without().
9623 : */
9624 :
9625 : #ifdef CONFIG_SMP
9626 : if (rq->ttwu_pending)
9627 : return 0;
9628 : #endif
9629 :
9630 : return 1;
9631 : }
9632 :
9633 : /*
9634 : * update_sg_wakeup_stats - Update sched_group's statistics for wakeup.
9635 : * @sd: The sched_domain level to look for idlest group.
9636 : * @group: sched_group whose statistics are to be updated.
9637 : * @sgs: variable to hold the statistics for this group.
9638 : * @p: The task for which we look for the idlest group/CPU.
9639 : */
9640 : static inline void update_sg_wakeup_stats(struct sched_domain *sd,
9641 : struct sched_group *group,
9642 : struct sg_lb_stats *sgs,
9643 : struct task_struct *p)
9644 : {
9645 : int i, nr_running;
9646 :
9647 : memset(sgs, 0, sizeof(*sgs));
9648 :
9649 : /* Assume that task can't fit any CPU of the group */
9650 : if (sd->flags & SD_ASYM_CPUCAPACITY)
9651 : sgs->group_misfit_task_load = 1;
9652 :
9653 : for_each_cpu(i, sched_group_span(group)) {
9654 : struct rq *rq = cpu_rq(i);
9655 : unsigned int local;
9656 :
9657 : sgs->group_load += cpu_load_without(rq, p);
9658 : sgs->group_util += cpu_util_without(i, p);
9659 : sgs->group_runnable += cpu_runnable_without(rq, p);
9660 : local = task_running_on_cpu(i, p);
9661 : sgs->sum_h_nr_running += rq->cfs.h_nr_running - local;
9662 :
9663 : nr_running = rq->nr_running - local;
9664 : sgs->sum_nr_running += nr_running;
9665 :
9666 : /*
9667 : * No need to call idle_cpu_without() if nr_running is not 0
9668 : */
9669 : if (!nr_running && idle_cpu_without(i, p))
9670 : sgs->idle_cpus++;
9671 :
9672 : /* Check if task fits in the CPU */
9673 : if (sd->flags & SD_ASYM_CPUCAPACITY &&
9674 : sgs->group_misfit_task_load &&
9675 : task_fits_cpu(p, i))
9676 : sgs->group_misfit_task_load = 0;
9677 :
9678 : }
9679 :
9680 : sgs->group_capacity = group->sgc->capacity;
9681 :
9682 : sgs->group_weight = group->group_weight;
9683 :
9684 : sgs->group_type = group_classify(sd->imbalance_pct, group, sgs);
9685 :
9686 : /*
9687 : * Computing avg_load makes sense only when group is fully busy or
9688 : * overloaded
9689 : */
9690 : if (sgs->group_type == group_fully_busy ||
9691 : sgs->group_type == group_overloaded)
9692 : sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
9693 : sgs->group_capacity;
9694 : }
9695 :
9696 : static bool update_pick_idlest(struct sched_group *idlest,
9697 : struct sg_lb_stats *idlest_sgs,
9698 : struct sched_group *group,
9699 : struct sg_lb_stats *sgs)
9700 : {
9701 : if (sgs->group_type < idlest_sgs->group_type)
9702 : return true;
9703 :
9704 : if (sgs->group_type > idlest_sgs->group_type)
9705 : return false;
9706 :
9707 : /*
9708 : * The candidate and the current idlest group are the same type of
9709 : * group. Let check which one is the idlest according to the type.
9710 : */
9711 :
9712 : switch (sgs->group_type) {
9713 : case group_overloaded:
9714 : case group_fully_busy:
9715 : /* Select the group with lowest avg_load. */
9716 : if (idlest_sgs->avg_load <= sgs->avg_load)
9717 : return false;
9718 : break;
9719 :
9720 : case group_imbalanced:
9721 : case group_asym_packing:
9722 : /* Those types are not used in the slow wakeup path */
9723 : return false;
9724 :
9725 : case group_misfit_task:
9726 : /* Select group with the highest max capacity */
9727 : if (idlest->sgc->max_capacity >= group->sgc->max_capacity)
9728 : return false;
9729 : break;
9730 :
9731 : case group_has_spare:
9732 : /* Select group with most idle CPUs */
9733 : if (idlest_sgs->idle_cpus > sgs->idle_cpus)
9734 : return false;
9735 :
9736 : /* Select group with lowest group_util */
9737 : if (idlest_sgs->idle_cpus == sgs->idle_cpus &&
9738 : idlest_sgs->group_util <= sgs->group_util)
9739 : return false;
9740 :
9741 : break;
9742 : }
9743 :
9744 : return true;
9745 : }
9746 :
9747 : /*
9748 : * find_idlest_group() finds and returns the least busy CPU group within the
9749 : * domain.
9750 : *
9751 : * Assumes p is allowed on at least one CPU in sd.
9752 : */
9753 : static struct sched_group *
9754 : find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
9755 : {
9756 : struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups;
9757 : struct sg_lb_stats local_sgs, tmp_sgs;
9758 : struct sg_lb_stats *sgs;
9759 : unsigned long imbalance;
9760 : struct sg_lb_stats idlest_sgs = {
9761 : .avg_load = UINT_MAX,
9762 : .group_type = group_overloaded,
9763 : };
9764 :
9765 : do {
9766 : int local_group;
9767 :
9768 : /* Skip over this group if it has no CPUs allowed */
9769 : if (!cpumask_intersects(sched_group_span(group),
9770 : p->cpus_ptr))
9771 : continue;
9772 :
9773 : /* Skip over this group if no cookie matched */
9774 : if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))
9775 : continue;
9776 :
9777 : local_group = cpumask_test_cpu(this_cpu,
9778 : sched_group_span(group));
9779 :
9780 : if (local_group) {
9781 : sgs = &local_sgs;
9782 : local = group;
9783 : } else {
9784 : sgs = &tmp_sgs;
9785 : }
9786 :
9787 : update_sg_wakeup_stats(sd, group, sgs, p);
9788 :
9789 : if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) {
9790 : idlest = group;
9791 : idlest_sgs = *sgs;
9792 : }
9793 :
9794 : } while (group = group->next, group != sd->groups);
9795 :
9796 :
9797 : /* There is no idlest group to push tasks to */
9798 : if (!idlest)
9799 : return NULL;
9800 :
9801 : /* The local group has been skipped because of CPU affinity */
9802 : if (!local)
9803 : return idlest;
9804 :
9805 : /*
9806 : * If the local group is idler than the selected idlest group
9807 : * don't try and push the task.
9808 : */
9809 : if (local_sgs.group_type < idlest_sgs.group_type)
9810 : return NULL;
9811 :
9812 : /*
9813 : * If the local group is busier than the selected idlest group
9814 : * try and push the task.
9815 : */
9816 : if (local_sgs.group_type > idlest_sgs.group_type)
9817 : return idlest;
9818 :
9819 : switch (local_sgs.group_type) {
9820 : case group_overloaded:
9821 : case group_fully_busy:
9822 :
9823 : /* Calculate allowed imbalance based on load */
9824 : imbalance = scale_load_down(NICE_0_LOAD) *
9825 : (sd->imbalance_pct-100) / 100;
9826 :
9827 : /*
9828 : * When comparing groups across NUMA domains, it's possible for
9829 : * the local domain to be very lightly loaded relative to the
9830 : * remote domains but "imbalance" skews the comparison making
9831 : * remote CPUs look much more favourable. When considering
9832 : * cross-domain, add imbalance to the load on the remote node
9833 : * and consider staying local.
9834 : */
9835 :
9836 : if ((sd->flags & SD_NUMA) &&
9837 : ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load))
9838 : return NULL;
9839 :
9840 : /*
9841 : * If the local group is less loaded than the selected
9842 : * idlest group don't try and push any tasks.
9843 : */
9844 : if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance))
9845 : return NULL;
9846 :
9847 : if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load)
9848 : return NULL;
9849 : break;
9850 :
9851 : case group_imbalanced:
9852 : case group_asym_packing:
9853 : /* Those type are not used in the slow wakeup path */
9854 : return NULL;
9855 :
9856 : case group_misfit_task:
9857 : /* Select group with the highest max capacity */
9858 : if (local->sgc->max_capacity >= idlest->sgc->max_capacity)
9859 : return NULL;
9860 : break;
9861 :
9862 : case group_has_spare:
9863 : #ifdef CONFIG_NUMA
9864 : if (sd->flags & SD_NUMA) {
9865 : int imb_numa_nr = sd->imb_numa_nr;
9866 : #ifdef CONFIG_NUMA_BALANCING
9867 : int idlest_cpu;
9868 : /*
9869 : * If there is spare capacity at NUMA, try to select
9870 : * the preferred node
9871 : */
9872 : if (cpu_to_node(this_cpu) == p->numa_preferred_nid)
9873 : return NULL;
9874 :
9875 : idlest_cpu = cpumask_first(sched_group_span(idlest));
9876 : if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid)
9877 : return idlest;
9878 : #endif /* CONFIG_NUMA_BALANCING */
9879 : /*
9880 : * Otherwise, keep the task close to the wakeup source
9881 : * and improve locality if the number of running tasks
9882 : * would remain below threshold where an imbalance is
9883 : * allowed while accounting for the possibility the
9884 : * task is pinned to a subset of CPUs. If there is a
9885 : * real need of migration, periodic load balance will
9886 : * take care of it.
9887 : */
9888 : if (p->nr_cpus_allowed != NR_CPUS) {
9889 : struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
9890 :
9891 : cpumask_and(cpus, sched_group_span(local), p->cpus_ptr);
9892 : imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr);
9893 : }
9894 :
9895 : imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus);
9896 : if (!adjust_numa_imbalance(imbalance,
9897 : local_sgs.sum_nr_running + 1,
9898 : imb_numa_nr)) {
9899 : return NULL;
9900 : }
9901 : }
9902 : #endif /* CONFIG_NUMA */
9903 :
9904 : /*
9905 : * Select group with highest number of idle CPUs. We could also
9906 : * compare the utilization which is more stable but it can end
9907 : * up that the group has less spare capacity but finally more
9908 : * idle CPUs which means more opportunity to run task.
9909 : */
9910 : if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus)
9911 : return NULL;
9912 : break;
9913 : }
9914 :
9915 : return idlest;
9916 : }
9917 :
9918 : static void update_idle_cpu_scan(struct lb_env *env,
9919 : unsigned long sum_util)
9920 : {
9921 : struct sched_domain_shared *sd_share;
9922 : int llc_weight, pct;
9923 : u64 x, y, tmp;
9924 : /*
9925 : * Update the number of CPUs to scan in LLC domain, which could
9926 : * be used as a hint in select_idle_cpu(). The update of sd_share
9927 : * could be expensive because it is within a shared cache line.
9928 : * So the write of this hint only occurs during periodic load
9929 : * balancing, rather than CPU_NEWLY_IDLE, because the latter
9930 : * can fire way more frequently than the former.
9931 : */
9932 : if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE)
9933 : return;
9934 :
9935 : llc_weight = per_cpu(sd_llc_size, env->dst_cpu);
9936 : if (env->sd->span_weight != llc_weight)
9937 : return;
9938 :
9939 : sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu));
9940 : if (!sd_share)
9941 : return;
9942 :
9943 : /*
9944 : * The number of CPUs to search drops as sum_util increases, when
9945 : * sum_util hits 85% or above, the scan stops.
9946 : * The reason to choose 85% as the threshold is because this is the
9947 : * imbalance_pct(117) when a LLC sched group is overloaded.
9948 : *
9949 : * let y = SCHED_CAPACITY_SCALE - p * x^2 [1]
9950 : * and y'= y / SCHED_CAPACITY_SCALE
9951 : *
9952 : * x is the ratio of sum_util compared to the CPU capacity:
9953 : * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE)
9954 : * y' is the ratio of CPUs to be scanned in the LLC domain,
9955 : * and the number of CPUs to scan is calculated by:
9956 : *
9957 : * nr_scan = llc_weight * y' [2]
9958 : *
9959 : * When x hits the threshold of overloaded, AKA, when
9960 : * x = 100 / pct, y drops to 0. According to [1],
9961 : * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000
9962 : *
9963 : * Scale x by SCHED_CAPACITY_SCALE:
9964 : * x' = sum_util / llc_weight; [3]
9965 : *
9966 : * and finally [1] becomes:
9967 : * y = SCHED_CAPACITY_SCALE -
9968 : * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4]
9969 : *
9970 : */
9971 : /* equation [3] */
9972 : x = sum_util;
9973 : do_div(x, llc_weight);
9974 :
9975 : /* equation [4] */
9976 : pct = env->sd->imbalance_pct;
9977 : tmp = x * x * pct * pct;
9978 : do_div(tmp, 10000 * SCHED_CAPACITY_SCALE);
9979 : tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE);
9980 : y = SCHED_CAPACITY_SCALE - tmp;
9981 :
9982 : /* equation [2] */
9983 : y *= llc_weight;
9984 : do_div(y, SCHED_CAPACITY_SCALE);
9985 : if ((int)y != sd_share->nr_idle_scan)
9986 : WRITE_ONCE(sd_share->nr_idle_scan, (int)y);
9987 : }
9988 :
9989 : /**
9990 : * update_sd_lb_stats - Update sched_domain's statistics for load balancing.
9991 : * @env: The load balancing environment.
9992 : * @sds: variable to hold the statistics for this sched_domain.
9993 : */
9994 :
9995 : static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
9996 : {
9997 : struct sched_domain *child = env->sd->child;
9998 : struct sched_group *sg = env->sd->groups;
9999 : struct sg_lb_stats *local = &sds->local_stat;
10000 : struct sg_lb_stats tmp_sgs;
10001 : unsigned long sum_util = 0;
10002 : int sg_status = 0;
10003 :
10004 : do {
10005 : struct sg_lb_stats *sgs = &tmp_sgs;
10006 : int local_group;
10007 :
10008 : local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));
10009 : if (local_group) {
10010 : sds->local = sg;
10011 : sgs = local;
10012 :
10013 : if (env->idle != CPU_NEWLY_IDLE ||
10014 : time_after_eq(jiffies, sg->sgc->next_update))
10015 : update_group_capacity(env->sd, env->dst_cpu);
10016 : }
10017 :
10018 : update_sg_lb_stats(env, sds, sg, sgs, &sg_status);
10019 :
10020 : if (local_group)
10021 : goto next_group;
10022 :
10023 :
10024 : if (update_sd_pick_busiest(env, sds, sg, sgs)) {
10025 : sds->busiest = sg;
10026 : sds->busiest_stat = *sgs;
10027 : }
10028 :
10029 : next_group:
10030 : /* Now, start updating sd_lb_stats */
10031 : sds->total_load += sgs->group_load;
10032 : sds->total_capacity += sgs->group_capacity;
10033 :
10034 : sum_util += sgs->group_util;
10035 : sg = sg->next;
10036 : } while (sg != env->sd->groups);
10037 :
10038 : /* Tag domain that child domain prefers tasks go to siblings first */
10039 : sds->prefer_sibling = child && child->flags & SD_PREFER_SIBLING;
10040 :
10041 :
10042 : if (env->sd->flags & SD_NUMA)
10043 : env->fbq_type = fbq_classify_group(&sds->busiest_stat);
10044 :
10045 : if (!env->sd->parent) {
10046 : struct root_domain *rd = env->dst_rq->rd;
10047 :
10048 : /* update overload indicator if we are at root domain */
10049 : WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD);
10050 :
10051 : /* Update over-utilization (tipping point, U >= 0) indicator */
10052 : WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED);
10053 : trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED);
10054 : } else if (sg_status & SG_OVERUTILIZED) {
10055 : struct root_domain *rd = env->dst_rq->rd;
10056 :
10057 : WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED);
10058 : trace_sched_overutilized_tp(rd, SG_OVERUTILIZED);
10059 : }
10060 :
10061 : update_idle_cpu_scan(env, sum_util);
10062 : }
10063 :
10064 : /**
10065 : * calculate_imbalance - Calculate the amount of imbalance present within the
10066 : * groups of a given sched_domain during load balance.
10067 : * @env: load balance environment
10068 : * @sds: statistics of the sched_domain whose imbalance is to be calculated.
10069 : */
10070 : static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
10071 : {
10072 : struct sg_lb_stats *local, *busiest;
10073 :
10074 : local = &sds->local_stat;
10075 : busiest = &sds->busiest_stat;
10076 :
10077 : if (busiest->group_type == group_misfit_task) {
10078 : if (env->sd->flags & SD_ASYM_CPUCAPACITY) {
10079 : /* Set imbalance to allow misfit tasks to be balanced. */
10080 : env->migration_type = migrate_misfit;
10081 : env->imbalance = 1;
10082 : } else {
10083 : /*
10084 : * Set load imbalance to allow moving task from cpu
10085 : * with reduced capacity.
10086 : */
10087 : env->migration_type = migrate_load;
10088 : env->imbalance = busiest->group_misfit_task_load;
10089 : }
10090 : return;
10091 : }
10092 :
10093 : if (busiest->group_type == group_asym_packing) {
10094 : /*
10095 : * In case of asym capacity, we will try to migrate all load to
10096 : * the preferred CPU.
10097 : */
10098 : env->migration_type = migrate_task;
10099 : env->imbalance = busiest->sum_h_nr_running;
10100 : return;
10101 : }
10102 :
10103 : if (busiest->group_type == group_imbalanced) {
10104 : /*
10105 : * In the group_imb case we cannot rely on group-wide averages
10106 : * to ensure CPU-load equilibrium, try to move any task to fix
10107 : * the imbalance. The next load balance will take care of
10108 : * balancing back the system.
10109 : */
10110 : env->migration_type = migrate_task;
10111 : env->imbalance = 1;
10112 : return;
10113 : }
10114 :
10115 : /*
10116 : * Try to use spare capacity of local group without overloading it or
10117 : * emptying busiest.
10118 : */
10119 : if (local->group_type == group_has_spare) {
10120 : if ((busiest->group_type > group_fully_busy) &&
10121 : !(env->sd->flags & SD_SHARE_PKG_RESOURCES)) {
10122 : /*
10123 : * If busiest is overloaded, try to fill spare
10124 : * capacity. This might end up creating spare capacity
10125 : * in busiest or busiest still being overloaded but
10126 : * there is no simple way to directly compute the
10127 : * amount of load to migrate in order to balance the
10128 : * system.
10129 : */
10130 : env->migration_type = migrate_util;
10131 : env->imbalance = max(local->group_capacity, local->group_util) -
10132 : local->group_util;
10133 :
10134 : /*
10135 : * In some cases, the group's utilization is max or even
10136 : * higher than capacity because of migrations but the
10137 : * local CPU is (newly) idle. There is at least one
10138 : * waiting task in this overloaded busiest group. Let's
10139 : * try to pull it.
10140 : */
10141 : if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) {
10142 : env->migration_type = migrate_task;
10143 : env->imbalance = 1;
10144 : }
10145 :
10146 : return;
10147 : }
10148 :
10149 : if (busiest->group_weight == 1 || sds->prefer_sibling) {
10150 : unsigned int nr_diff = busiest->sum_nr_running;
10151 : /*
10152 : * When prefer sibling, evenly spread running tasks on
10153 : * groups.
10154 : */
10155 : env->migration_type = migrate_task;
10156 : lsub_positive(&nr_diff, local->sum_nr_running);
10157 : env->imbalance = nr_diff;
10158 : } else {
10159 :
10160 : /*
10161 : * If there is no overload, we just want to even the number of
10162 : * idle cpus.
10163 : */
10164 : env->migration_type = migrate_task;
10165 : env->imbalance = max_t(long, 0,
10166 : (local->idle_cpus - busiest->idle_cpus));
10167 : }
10168 :
10169 : #ifdef CONFIG_NUMA
10170 : /* Consider allowing a small imbalance between NUMA groups */
10171 : if (env->sd->flags & SD_NUMA) {
10172 : env->imbalance = adjust_numa_imbalance(env->imbalance,
10173 : local->sum_nr_running + 1,
10174 : env->sd->imb_numa_nr);
10175 : }
10176 : #endif
10177 :
10178 : /* Number of tasks to move to restore balance */
10179 : env->imbalance >>= 1;
10180 :
10181 : return;
10182 : }
10183 :
10184 : /*
10185 : * Local is fully busy but has to take more load to relieve the
10186 : * busiest group
10187 : */
10188 : if (local->group_type < group_overloaded) {
10189 : /*
10190 : * Local will become overloaded so the avg_load metrics are
10191 : * finally needed.
10192 : */
10193 :
10194 : local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
10195 : local->group_capacity;
10196 :
10197 : /*
10198 : * If the local group is more loaded than the selected
10199 : * busiest group don't try to pull any tasks.
10200 : */
10201 : if (local->avg_load >= busiest->avg_load) {
10202 : env->imbalance = 0;
10203 : return;
10204 : }
10205 :
10206 : sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
10207 : sds->total_capacity;
10208 : }
10209 :
10210 : /*
10211 : * Both group are or will become overloaded and we're trying to get all
10212 : * the CPUs to the average_load, so we don't want to push ourselves
10213 : * above the average load, nor do we wish to reduce the max loaded CPU
10214 : * below the average load. At the same time, we also don't want to
10215 : * reduce the group load below the group capacity. Thus we look for
10216 : * the minimum possible imbalance.
10217 : */
10218 : env->migration_type = migrate_load;
10219 : env->imbalance = min(
10220 : (busiest->avg_load - sds->avg_load) * busiest->group_capacity,
10221 : (sds->avg_load - local->avg_load) * local->group_capacity
10222 : ) / SCHED_CAPACITY_SCALE;
10223 : }
10224 :
10225 : /******* find_busiest_group() helpers end here *********************/
10226 :
10227 : /*
10228 : * Decision matrix according to the local and busiest group type:
10229 : *
10230 : * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
10231 : * has_spare nr_idle balanced N/A N/A balanced balanced
10232 : * fully_busy nr_idle nr_idle N/A N/A balanced balanced
10233 : * misfit_task force N/A N/A N/A N/A N/A
10234 : * asym_packing force force N/A N/A force force
10235 : * imbalanced force force N/A N/A force force
10236 : * overloaded force force N/A N/A force avg_load
10237 : *
10238 : * N/A : Not Applicable because already filtered while updating
10239 : * statistics.
10240 : * balanced : The system is balanced for these 2 groups.
10241 : * force : Calculate the imbalance as load migration is probably needed.
10242 : * avg_load : Only if imbalance is significant enough.
10243 : * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite
10244 : * different in groups.
10245 : */
10246 :
10247 : /**
10248 : * find_busiest_group - Returns the busiest group within the sched_domain
10249 : * if there is an imbalance.
10250 : * @env: The load balancing environment.
10251 : *
10252 : * Also calculates the amount of runnable load which should be moved
10253 : * to restore balance.
10254 : *
10255 : * Return: - The busiest group if imbalance exists.
10256 : */
10257 : static struct sched_group *find_busiest_group(struct lb_env *env)
10258 : {
10259 : struct sg_lb_stats *local, *busiest;
10260 : struct sd_lb_stats sds;
10261 :
10262 : init_sd_lb_stats(&sds);
10263 :
10264 : /*
10265 : * Compute the various statistics relevant for load balancing at
10266 : * this level.
10267 : */
10268 : update_sd_lb_stats(env, &sds);
10269 :
10270 : /* There is no busy sibling group to pull tasks from */
10271 : if (!sds.busiest)
10272 : goto out_balanced;
10273 :
10274 : busiest = &sds.busiest_stat;
10275 :
10276 : /* Misfit tasks should be dealt with regardless of the avg load */
10277 : if (busiest->group_type == group_misfit_task)
10278 : goto force_balance;
10279 :
10280 : if (sched_energy_enabled()) {
10281 : struct root_domain *rd = env->dst_rq->rd;
10282 :
10283 : if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized))
10284 : goto out_balanced;
10285 : }
10286 :
10287 : /* ASYM feature bypasses nice load balance check */
10288 : if (busiest->group_type == group_asym_packing)
10289 : goto force_balance;
10290 :
10291 : /*
10292 : * If the busiest group is imbalanced the below checks don't
10293 : * work because they assume all things are equal, which typically
10294 : * isn't true due to cpus_ptr constraints and the like.
10295 : */
10296 : if (busiest->group_type == group_imbalanced)
10297 : goto force_balance;
10298 :
10299 : local = &sds.local_stat;
10300 : /*
10301 : * If the local group is busier than the selected busiest group
10302 : * don't try and pull any tasks.
10303 : */
10304 : if (local->group_type > busiest->group_type)
10305 : goto out_balanced;
10306 :
10307 : /*
10308 : * When groups are overloaded, use the avg_load to ensure fairness
10309 : * between tasks.
10310 : */
10311 : if (local->group_type == group_overloaded) {
10312 : /*
10313 : * If the local group is more loaded than the selected
10314 : * busiest group don't try to pull any tasks.
10315 : */
10316 : if (local->avg_load >= busiest->avg_load)
10317 : goto out_balanced;
10318 :
10319 : /* XXX broken for overlapping NUMA groups */
10320 : sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) /
10321 : sds.total_capacity;
10322 :
10323 : /*
10324 : * Don't pull any tasks if this group is already above the
10325 : * domain average load.
10326 : */
10327 : if (local->avg_load >= sds.avg_load)
10328 : goto out_balanced;
10329 :
10330 : /*
10331 : * If the busiest group is more loaded, use imbalance_pct to be
10332 : * conservative.
10333 : */
10334 : if (100 * busiest->avg_load <=
10335 : env->sd->imbalance_pct * local->avg_load)
10336 : goto out_balanced;
10337 : }
10338 :
10339 : /* Try to move all excess tasks to child's sibling domain */
10340 : if (sds.prefer_sibling && local->group_type == group_has_spare &&
10341 : busiest->sum_nr_running > local->sum_nr_running + 1)
10342 : goto force_balance;
10343 :
10344 : if (busiest->group_type != group_overloaded) {
10345 : if (env->idle == CPU_NOT_IDLE)
10346 : /*
10347 : * If the busiest group is not overloaded (and as a
10348 : * result the local one too) but this CPU is already
10349 : * busy, let another idle CPU try to pull task.
10350 : */
10351 : goto out_balanced;
10352 :
10353 : if (busiest->group_weight > 1 &&
10354 : local->idle_cpus <= (busiest->idle_cpus + 1))
10355 : /*
10356 : * If the busiest group is not overloaded
10357 : * and there is no imbalance between this and busiest
10358 : * group wrt idle CPUs, it is balanced. The imbalance
10359 : * becomes significant if the diff is greater than 1
10360 : * otherwise we might end up to just move the imbalance
10361 : * on another group. Of course this applies only if
10362 : * there is more than 1 CPU per group.
10363 : */
10364 : goto out_balanced;
10365 :
10366 : if (busiest->sum_h_nr_running == 1)
10367 : /*
10368 : * busiest doesn't have any tasks waiting to run
10369 : */
10370 : goto out_balanced;
10371 : }
10372 :
10373 : force_balance:
10374 : /* Looks like there is an imbalance. Compute it */
10375 : calculate_imbalance(env, &sds);
10376 : return env->imbalance ? sds.busiest : NULL;
10377 :
10378 : out_balanced:
10379 : env->imbalance = 0;
10380 : return NULL;
10381 : }
10382 :
10383 : /*
10384 : * find_busiest_queue - find the busiest runqueue among the CPUs in the group.
10385 : */
10386 : static struct rq *find_busiest_queue(struct lb_env *env,
10387 : struct sched_group *group)
10388 : {
10389 : struct rq *busiest = NULL, *rq;
10390 : unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
10391 : unsigned int busiest_nr = 0;
10392 : int i;
10393 :
10394 : for_each_cpu_and(i, sched_group_span(group), env->cpus) {
10395 : unsigned long capacity, load, util;
10396 : unsigned int nr_running;
10397 : enum fbq_type rt;
10398 :
10399 : rq = cpu_rq(i);
10400 : rt = fbq_classify_rq(rq);
10401 :
10402 : /*
10403 : * We classify groups/runqueues into three groups:
10404 : * - regular: there are !numa tasks
10405 : * - remote: there are numa tasks that run on the 'wrong' node
10406 : * - all: there is no distinction
10407 : *
10408 : * In order to avoid migrating ideally placed numa tasks,
10409 : * ignore those when there's better options.
10410 : *
10411 : * If we ignore the actual busiest queue to migrate another
10412 : * task, the next balance pass can still reduce the busiest
10413 : * queue by moving tasks around inside the node.
10414 : *
10415 : * If we cannot move enough load due to this classification
10416 : * the next pass will adjust the group classification and
10417 : * allow migration of more tasks.
10418 : *
10419 : * Both cases only affect the total convergence complexity.
10420 : */
10421 : if (rt > env->fbq_type)
10422 : continue;
10423 :
10424 : nr_running = rq->cfs.h_nr_running;
10425 : if (!nr_running)
10426 : continue;
10427 :
10428 : capacity = capacity_of(i);
10429 :
10430 : /*
10431 : * For ASYM_CPUCAPACITY domains, don't pick a CPU that could
10432 : * eventually lead to active_balancing high->low capacity.
10433 : * Higher per-CPU capacity is considered better than balancing
10434 : * average load.
10435 : */
10436 : if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
10437 : !capacity_greater(capacity_of(env->dst_cpu), capacity) &&
10438 : nr_running == 1)
10439 : continue;
10440 :
10441 : /* Make sure we only pull tasks from a CPU of lower priority */
10442 : if ((env->sd->flags & SD_ASYM_PACKING) &&
10443 : sched_asym_prefer(i, env->dst_cpu) &&
10444 : nr_running == 1)
10445 : continue;
10446 :
10447 : switch (env->migration_type) {
10448 : case migrate_load:
10449 : /*
10450 : * When comparing with load imbalance, use cpu_load()
10451 : * which is not scaled with the CPU capacity.
10452 : */
10453 : load = cpu_load(rq);
10454 :
10455 : if (nr_running == 1 && load > env->imbalance &&
10456 : !check_cpu_capacity(rq, env->sd))
10457 : break;
10458 :
10459 : /*
10460 : * For the load comparisons with the other CPUs,
10461 : * consider the cpu_load() scaled with the CPU
10462 : * capacity, so that the load can be moved away
10463 : * from the CPU that is potentially running at a
10464 : * lower capacity.
10465 : *
10466 : * Thus we're looking for max(load_i / capacity_i),
10467 : * crosswise multiplication to rid ourselves of the
10468 : * division works out to:
10469 : * load_i * capacity_j > load_j * capacity_i;
10470 : * where j is our previous maximum.
10471 : */
10472 : if (load * busiest_capacity > busiest_load * capacity) {
10473 : busiest_load = load;
10474 : busiest_capacity = capacity;
10475 : busiest = rq;
10476 : }
10477 : break;
10478 :
10479 : case migrate_util:
10480 : util = cpu_util_cfs(i);
10481 :
10482 : /*
10483 : * Don't try to pull utilization from a CPU with one
10484 : * running task. Whatever its utilization, we will fail
10485 : * detach the task.
10486 : */
10487 : if (nr_running <= 1)
10488 : continue;
10489 :
10490 : if (busiest_util < util) {
10491 : busiest_util = util;
10492 : busiest = rq;
10493 : }
10494 : break;
10495 :
10496 : case migrate_task:
10497 : if (busiest_nr < nr_running) {
10498 : busiest_nr = nr_running;
10499 : busiest = rq;
10500 : }
10501 : break;
10502 :
10503 : case migrate_misfit:
10504 : /*
10505 : * For ASYM_CPUCAPACITY domains with misfit tasks we
10506 : * simply seek the "biggest" misfit task.
10507 : */
10508 : if (rq->misfit_task_load > busiest_load) {
10509 : busiest_load = rq->misfit_task_load;
10510 : busiest = rq;
10511 : }
10512 :
10513 : break;
10514 :
10515 : }
10516 : }
10517 :
10518 : return busiest;
10519 : }
10520 :
10521 : /*
10522 : * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
10523 : * so long as it is large enough.
10524 : */
10525 : #define MAX_PINNED_INTERVAL 512
10526 :
10527 : static inline bool
10528 : asym_active_balance(struct lb_env *env)
10529 : {
10530 : /*
10531 : * ASYM_PACKING needs to force migrate tasks from busy but
10532 : * lower priority CPUs in order to pack all tasks in the
10533 : * highest priority CPUs.
10534 : */
10535 : return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) &&
10536 : sched_asym_prefer(env->dst_cpu, env->src_cpu);
10537 : }
10538 :
10539 : static inline bool
10540 : imbalanced_active_balance(struct lb_env *env)
10541 : {
10542 : struct sched_domain *sd = env->sd;
10543 :
10544 : /*
10545 : * The imbalanced case includes the case of pinned tasks preventing a fair
10546 : * distribution of the load on the system but also the even distribution of the
10547 : * threads on a system with spare capacity
10548 : */
10549 : if ((env->migration_type == migrate_task) &&
10550 : (sd->nr_balance_failed > sd->cache_nice_tries+2))
10551 : return 1;
10552 :
10553 : return 0;
10554 : }
10555 :
10556 : static int need_active_balance(struct lb_env *env)
10557 : {
10558 : struct sched_domain *sd = env->sd;
10559 :
10560 : if (asym_active_balance(env))
10561 : return 1;
10562 :
10563 : if (imbalanced_active_balance(env))
10564 : return 1;
10565 :
10566 : /*
10567 : * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task.
10568 : * It's worth migrating the task if the src_cpu's capacity is reduced
10569 : * because of other sched_class or IRQs if more capacity stays
10570 : * available on dst_cpu.
10571 : */
10572 : if ((env->idle != CPU_NOT_IDLE) &&
10573 : (env->src_rq->cfs.h_nr_running == 1)) {
10574 : if ((check_cpu_capacity(env->src_rq, sd)) &&
10575 : (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))
10576 : return 1;
10577 : }
10578 :
10579 : if (env->migration_type == migrate_misfit)
10580 : return 1;
10581 :
10582 : return 0;
10583 : }
10584 :
10585 : static int active_load_balance_cpu_stop(void *data);
10586 :
10587 : static int should_we_balance(struct lb_env *env)
10588 : {
10589 : struct sched_group *sg = env->sd->groups;
10590 : int cpu;
10591 :
10592 : /*
10593 : * Ensure the balancing environment is consistent; can happen
10594 : * when the softirq triggers 'during' hotplug.
10595 : */
10596 : if (!cpumask_test_cpu(env->dst_cpu, env->cpus))
10597 : return 0;
10598 :
10599 : /*
10600 : * In the newly idle case, we will allow all the CPUs
10601 : * to do the newly idle load balance.
10602 : *
10603 : * However, we bail out if we already have tasks or a wakeup pending,
10604 : * to optimize wakeup latency.
10605 : */
10606 : if (env->idle == CPU_NEWLY_IDLE) {
10607 : if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending)
10608 : return 0;
10609 : return 1;
10610 : }
10611 :
10612 : /* Try to find first idle CPU */
10613 : for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) {
10614 : if (!idle_cpu(cpu))
10615 : continue;
10616 :
10617 : /* Are we the first idle CPU? */
10618 : return cpu == env->dst_cpu;
10619 : }
10620 :
10621 : /* Are we the first CPU of this group ? */
10622 : return group_balance_cpu(sg) == env->dst_cpu;
10623 : }
10624 :
10625 : /*
10626 : * Check this_cpu to ensure it is balanced within domain. Attempt to move
10627 : * tasks if there is an imbalance.
10628 : */
10629 : static int load_balance(int this_cpu, struct rq *this_rq,
10630 : struct sched_domain *sd, enum cpu_idle_type idle,
10631 : int *continue_balancing)
10632 : {
10633 : int ld_moved, cur_ld_moved, active_balance = 0;
10634 : struct sched_domain *sd_parent = sd->parent;
10635 : struct sched_group *group;
10636 : struct rq *busiest;
10637 : struct rq_flags rf;
10638 : struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);
10639 : struct lb_env env = {
10640 : .sd = sd,
10641 : .dst_cpu = this_cpu,
10642 : .dst_rq = this_rq,
10643 : .dst_grpmask = sched_group_span(sd->groups),
10644 : .idle = idle,
10645 : .loop_break = SCHED_NR_MIGRATE_BREAK,
10646 : .cpus = cpus,
10647 : .fbq_type = all,
10648 : .tasks = LIST_HEAD_INIT(env.tasks),
10649 : };
10650 :
10651 : cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask);
10652 :
10653 : schedstat_inc(sd->lb_count[idle]);
10654 :
10655 : redo:
10656 : if (!should_we_balance(&env)) {
10657 : *continue_balancing = 0;
10658 : goto out_balanced;
10659 : }
10660 :
10661 : group = find_busiest_group(&env);
10662 : if (!group) {
10663 : schedstat_inc(sd->lb_nobusyg[idle]);
10664 : goto out_balanced;
10665 : }
10666 :
10667 : busiest = find_busiest_queue(&env, group);
10668 : if (!busiest) {
10669 : schedstat_inc(sd->lb_nobusyq[idle]);
10670 : goto out_balanced;
10671 : }
10672 :
10673 : WARN_ON_ONCE(busiest == env.dst_rq);
10674 :
10675 : schedstat_add(sd->lb_imbalance[idle], env.imbalance);
10676 :
10677 : env.src_cpu = busiest->cpu;
10678 : env.src_rq = busiest;
10679 :
10680 : ld_moved = 0;
10681 : /* Clear this flag as soon as we find a pullable task */
10682 : env.flags |= LBF_ALL_PINNED;
10683 : if (busiest->nr_running > 1) {
10684 : /*
10685 : * Attempt to move tasks. If find_busiest_group has found
10686 : * an imbalance but busiest->nr_running <= 1, the group is
10687 : * still unbalanced. ld_moved simply stays zero, so it is
10688 : * correctly treated as an imbalance.
10689 : */
10690 : env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
10691 :
10692 : more_balance:
10693 : rq_lock_irqsave(busiest, &rf);
10694 : update_rq_clock(busiest);
10695 :
10696 : /*
10697 : * cur_ld_moved - load moved in current iteration
10698 : * ld_moved - cumulative load moved across iterations
10699 : */
10700 : cur_ld_moved = detach_tasks(&env);
10701 :
10702 : /*
10703 : * We've detached some tasks from busiest_rq. Every
10704 : * task is masked "TASK_ON_RQ_MIGRATING", so we can safely
10705 : * unlock busiest->lock, and we are able to be sure
10706 : * that nobody can manipulate the tasks in parallel.
10707 : * See task_rq_lock() family for the details.
10708 : */
10709 :
10710 : rq_unlock(busiest, &rf);
10711 :
10712 : if (cur_ld_moved) {
10713 : attach_tasks(&env);
10714 : ld_moved += cur_ld_moved;
10715 : }
10716 :
10717 : local_irq_restore(rf.flags);
10718 :
10719 : if (env.flags & LBF_NEED_BREAK) {
10720 : env.flags &= ~LBF_NEED_BREAK;
10721 : /* Stop if we tried all running tasks */
10722 : if (env.loop < busiest->nr_running)
10723 : goto more_balance;
10724 : }
10725 :
10726 : /*
10727 : * Revisit (affine) tasks on src_cpu that couldn't be moved to
10728 : * us and move them to an alternate dst_cpu in our sched_group
10729 : * where they can run. The upper limit on how many times we
10730 : * iterate on same src_cpu is dependent on number of CPUs in our
10731 : * sched_group.
10732 : *
10733 : * This changes load balance semantics a bit on who can move
10734 : * load to a given_cpu. In addition to the given_cpu itself
10735 : * (or a ilb_cpu acting on its behalf where given_cpu is
10736 : * nohz-idle), we now have balance_cpu in a position to move
10737 : * load to given_cpu. In rare situations, this may cause
10738 : * conflicts (balance_cpu and given_cpu/ilb_cpu deciding
10739 : * _independently_ and at _same_ time to move some load to
10740 : * given_cpu) causing excess load to be moved to given_cpu.
10741 : * This however should not happen so much in practice and
10742 : * moreover subsequent load balance cycles should correct the
10743 : * excess load moved.
10744 : */
10745 : if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
10746 :
10747 : /* Prevent to re-select dst_cpu via env's CPUs */
10748 : __cpumask_clear_cpu(env.dst_cpu, env.cpus);
10749 :
10750 : env.dst_rq = cpu_rq(env.new_dst_cpu);
10751 : env.dst_cpu = env.new_dst_cpu;
10752 : env.flags &= ~LBF_DST_PINNED;
10753 : env.loop = 0;
10754 : env.loop_break = SCHED_NR_MIGRATE_BREAK;
10755 :
10756 : /*
10757 : * Go back to "more_balance" rather than "redo" since we
10758 : * need to continue with same src_cpu.
10759 : */
10760 : goto more_balance;
10761 : }
10762 :
10763 : /*
10764 : * We failed to reach balance because of affinity.
10765 : */
10766 : if (sd_parent) {
10767 : int *group_imbalance = &sd_parent->groups->sgc->imbalance;
10768 :
10769 : if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)
10770 : *group_imbalance = 1;
10771 : }
10772 :
10773 : /* All tasks on this runqueue were pinned by CPU affinity */
10774 : if (unlikely(env.flags & LBF_ALL_PINNED)) {
10775 : __cpumask_clear_cpu(cpu_of(busiest), cpus);
10776 : /*
10777 : * Attempting to continue load balancing at the current
10778 : * sched_domain level only makes sense if there are
10779 : * active CPUs remaining as possible busiest CPUs to
10780 : * pull load from which are not contained within the
10781 : * destination group that is receiving any migrated
10782 : * load.
10783 : */
10784 : if (!cpumask_subset(cpus, env.dst_grpmask)) {
10785 : env.loop = 0;
10786 : env.loop_break = SCHED_NR_MIGRATE_BREAK;
10787 : goto redo;
10788 : }
10789 : goto out_all_pinned;
10790 : }
10791 : }
10792 :
10793 : if (!ld_moved) {
10794 : schedstat_inc(sd->lb_failed[idle]);
10795 : /*
10796 : * Increment the failure counter only on periodic balance.
10797 : * We do not want newidle balance, which can be very
10798 : * frequent, pollute the failure counter causing
10799 : * excessive cache_hot migrations and active balances.
10800 : */
10801 : if (idle != CPU_NEWLY_IDLE)
10802 : sd->nr_balance_failed++;
10803 :
10804 : if (need_active_balance(&env)) {
10805 : unsigned long flags;
10806 :
10807 : raw_spin_rq_lock_irqsave(busiest, flags);
10808 :
10809 : /*
10810 : * Don't kick the active_load_balance_cpu_stop,
10811 : * if the curr task on busiest CPU can't be
10812 : * moved to this_cpu:
10813 : */
10814 : if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {
10815 : raw_spin_rq_unlock_irqrestore(busiest, flags);
10816 : goto out_one_pinned;
10817 : }
10818 :
10819 : /* Record that we found at least one task that could run on this_cpu */
10820 : env.flags &= ~LBF_ALL_PINNED;
10821 :
10822 : /*
10823 : * ->active_balance synchronizes accesses to
10824 : * ->active_balance_work. Once set, it's cleared
10825 : * only after active load balance is finished.
10826 : */
10827 : if (!busiest->active_balance) {
10828 : busiest->active_balance = 1;
10829 : busiest->push_cpu = this_cpu;
10830 : active_balance = 1;
10831 : }
10832 : raw_spin_rq_unlock_irqrestore(busiest, flags);
10833 :
10834 : if (active_balance) {
10835 : stop_one_cpu_nowait(cpu_of(busiest),
10836 : active_load_balance_cpu_stop, busiest,
10837 : &busiest->active_balance_work);
10838 : }
10839 : }
10840 : } else {
10841 : sd->nr_balance_failed = 0;
10842 : }
10843 :
10844 : if (likely(!active_balance) || need_active_balance(&env)) {
10845 : /* We were unbalanced, so reset the balancing interval */
10846 : sd->balance_interval = sd->min_interval;
10847 : }
10848 :
10849 : goto out;
10850 :
10851 : out_balanced:
10852 : /*
10853 : * We reach balance although we may have faced some affinity
10854 : * constraints. Clear the imbalance flag only if other tasks got
10855 : * a chance to move and fix the imbalance.
10856 : */
10857 : if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
10858 : int *group_imbalance = &sd_parent->groups->sgc->imbalance;
10859 :
10860 : if (*group_imbalance)
10861 : *group_imbalance = 0;
10862 : }
10863 :
10864 : out_all_pinned:
10865 : /*
10866 : * We reach balance because all tasks are pinned at this level so
10867 : * we can't migrate them. Let the imbalance flag set so parent level
10868 : * can try to migrate them.
10869 : */
10870 : schedstat_inc(sd->lb_balanced[idle]);
10871 :
10872 : sd->nr_balance_failed = 0;
10873 :
10874 : out_one_pinned:
10875 : ld_moved = 0;
10876 :
10877 : /*
10878 : * newidle_balance() disregards balance intervals, so we could
10879 : * repeatedly reach this code, which would lead to balance_interval
10880 : * skyrocketing in a short amount of time. Skip the balance_interval
10881 : * increase logic to avoid that.
10882 : */
10883 : if (env.idle == CPU_NEWLY_IDLE)
10884 : goto out;
10885 :
10886 : /* tune up the balancing interval */
10887 : if ((env.flags & LBF_ALL_PINNED &&
10888 : sd->balance_interval < MAX_PINNED_INTERVAL) ||
10889 : sd->balance_interval < sd->max_interval)
10890 : sd->balance_interval *= 2;
10891 : out:
10892 : return ld_moved;
10893 : }
10894 :
10895 : static inline unsigned long
10896 : get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)
10897 : {
10898 : unsigned long interval = sd->balance_interval;
10899 :
10900 : if (cpu_busy)
10901 : interval *= sd->busy_factor;
10902 :
10903 : /* scale ms to jiffies */
10904 : interval = msecs_to_jiffies(interval);
10905 :
10906 : /*
10907 : * Reduce likelihood of busy balancing at higher domains racing with
10908 : * balancing at lower domains by preventing their balancing periods
10909 : * from being multiples of each other.
10910 : */
10911 : if (cpu_busy)
10912 : interval -= 1;
10913 :
10914 : interval = clamp(interval, 1UL, max_load_balance_interval);
10915 :
10916 : return interval;
10917 : }
10918 :
10919 : static inline void
10920 : update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
10921 : {
10922 : unsigned long interval, next;
10923 :
10924 : /* used by idle balance, so cpu_busy = 0 */
10925 : interval = get_sd_balance_interval(sd, 0);
10926 : next = sd->last_balance + interval;
10927 :
10928 : if (time_after(*next_balance, next))
10929 : *next_balance = next;
10930 : }
10931 :
10932 : /*
10933 : * active_load_balance_cpu_stop is run by the CPU stopper. It pushes
10934 : * running tasks off the busiest CPU onto idle CPUs. It requires at
10935 : * least 1 task to be running on each physical CPU where possible, and
10936 : * avoids physical / logical imbalances.
10937 : */
10938 : static int active_load_balance_cpu_stop(void *data)
10939 : {
10940 : struct rq *busiest_rq = data;
10941 : int busiest_cpu = cpu_of(busiest_rq);
10942 : int target_cpu = busiest_rq->push_cpu;
10943 : struct rq *target_rq = cpu_rq(target_cpu);
10944 : struct sched_domain *sd;
10945 : struct task_struct *p = NULL;
10946 : struct rq_flags rf;
10947 :
10948 : rq_lock_irq(busiest_rq, &rf);
10949 : /*
10950 : * Between queueing the stop-work and running it is a hole in which
10951 : * CPUs can become inactive. We should not move tasks from or to
10952 : * inactive CPUs.
10953 : */
10954 : if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu))
10955 : goto out_unlock;
10956 :
10957 : /* Make sure the requested CPU hasn't gone down in the meantime: */
10958 : if (unlikely(busiest_cpu != smp_processor_id() ||
10959 : !busiest_rq->active_balance))
10960 : goto out_unlock;
10961 :
10962 : /* Is there any task to move? */
10963 : if (busiest_rq->nr_running <= 1)
10964 : goto out_unlock;
10965 :
10966 : /*
10967 : * This condition is "impossible", if it occurs
10968 : * we need to fix it. Originally reported by
10969 : * Bjorn Helgaas on a 128-CPU setup.
10970 : */
10971 : WARN_ON_ONCE(busiest_rq == target_rq);
10972 :
10973 : /* Search for an sd spanning us and the target CPU. */
10974 : rcu_read_lock();
10975 : for_each_domain(target_cpu, sd) {
10976 : if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
10977 : break;
10978 : }
10979 :
10980 : if (likely(sd)) {
10981 : struct lb_env env = {
10982 : .sd = sd,
10983 : .dst_cpu = target_cpu,
10984 : .dst_rq = target_rq,
10985 : .src_cpu = busiest_rq->cpu,
10986 : .src_rq = busiest_rq,
10987 : .idle = CPU_IDLE,
10988 : .flags = LBF_ACTIVE_LB,
10989 : };
10990 :
10991 : schedstat_inc(sd->alb_count);
10992 : update_rq_clock(busiest_rq);
10993 :
10994 : p = detach_one_task(&env);
10995 : if (p) {
10996 : schedstat_inc(sd->alb_pushed);
10997 : /* Active balancing done, reset the failure counter. */
10998 : sd->nr_balance_failed = 0;
10999 : } else {
11000 : schedstat_inc(sd->alb_failed);
11001 : }
11002 : }
11003 : rcu_read_unlock();
11004 : out_unlock:
11005 : busiest_rq->active_balance = 0;
11006 : rq_unlock(busiest_rq, &rf);
11007 :
11008 : if (p)
11009 : attach_one_task(target_rq, p);
11010 :
11011 : local_irq_enable();
11012 :
11013 : return 0;
11014 : }
11015 :
11016 : static DEFINE_SPINLOCK(balancing);
11017 :
11018 : /*
11019 : * Scale the max load_balance interval with the number of CPUs in the system.
11020 : * This trades load-balance latency on larger machines for less cross talk.
11021 : */
11022 : void update_max_interval(void)
11023 : {
11024 : max_load_balance_interval = HZ*num_online_cpus()/10;
11025 : }
11026 :
11027 : static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost)
11028 : {
11029 : if (cost > sd->max_newidle_lb_cost) {
11030 : /*
11031 : * Track max cost of a domain to make sure to not delay the
11032 : * next wakeup on the CPU.
11033 : */
11034 : sd->max_newidle_lb_cost = cost;
11035 : sd->last_decay_max_lb_cost = jiffies;
11036 : } else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) {
11037 : /*
11038 : * Decay the newidle max times by ~1% per second to ensure that
11039 : * it is not outdated and the current max cost is actually
11040 : * shorter.
11041 : */
11042 : sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256;
11043 : sd->last_decay_max_lb_cost = jiffies;
11044 :
11045 : return true;
11046 : }
11047 :
11048 : return false;
11049 : }
11050 :
11051 : /*
11052 : * It checks each scheduling domain to see if it is due to be balanced,
11053 : * and initiates a balancing operation if so.
11054 : *
11055 : * Balancing parameters are set up in init_sched_domains.
11056 : */
11057 : static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
11058 : {
11059 : int continue_balancing = 1;
11060 : int cpu = rq->cpu;
11061 : int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
11062 : unsigned long interval;
11063 : struct sched_domain *sd;
11064 : /* Earliest time when we have to do rebalance again */
11065 : unsigned long next_balance = jiffies + 60*HZ;
11066 : int update_next_balance = 0;
11067 : int need_serialize, need_decay = 0;
11068 : u64 max_cost = 0;
11069 :
11070 : rcu_read_lock();
11071 : for_each_domain(cpu, sd) {
11072 : /*
11073 : * Decay the newidle max times here because this is a regular
11074 : * visit to all the domains.
11075 : */
11076 : need_decay = update_newidle_cost(sd, 0);
11077 : max_cost += sd->max_newidle_lb_cost;
11078 :
11079 : /*
11080 : * Stop the load balance at this level. There is another
11081 : * CPU in our sched group which is doing load balancing more
11082 : * actively.
11083 : */
11084 : if (!continue_balancing) {
11085 : if (need_decay)
11086 : continue;
11087 : break;
11088 : }
11089 :
11090 : interval = get_sd_balance_interval(sd, busy);
11091 :
11092 : need_serialize = sd->flags & SD_SERIALIZE;
11093 : if (need_serialize) {
11094 : if (!spin_trylock(&balancing))
11095 : goto out;
11096 : }
11097 :
11098 : if (time_after_eq(jiffies, sd->last_balance + interval)) {
11099 : if (load_balance(cpu, rq, sd, idle, &continue_balancing)) {
11100 : /*
11101 : * The LBF_DST_PINNED logic could have changed
11102 : * env->dst_cpu, so we can't know our idle
11103 : * state even if we migrated tasks. Update it.
11104 : */
11105 : idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE;
11106 : busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
11107 : }
11108 : sd->last_balance = jiffies;
11109 : interval = get_sd_balance_interval(sd, busy);
11110 : }
11111 : if (need_serialize)
11112 : spin_unlock(&balancing);
11113 : out:
11114 : if (time_after(next_balance, sd->last_balance + interval)) {
11115 : next_balance = sd->last_balance + interval;
11116 : update_next_balance = 1;
11117 : }
11118 : }
11119 : if (need_decay) {
11120 : /*
11121 : * Ensure the rq-wide value also decays but keep it at a
11122 : * reasonable floor to avoid funnies with rq->avg_idle.
11123 : */
11124 : rq->max_idle_balance_cost =
11125 : max((u64)sysctl_sched_migration_cost, max_cost);
11126 : }
11127 : rcu_read_unlock();
11128 :
11129 : /*
11130 : * next_balance will be updated only when there is a need.
11131 : * When the cpu is attached to null domain for ex, it will not be
11132 : * updated.
11133 : */
11134 : if (likely(update_next_balance))
11135 : rq->next_balance = next_balance;
11136 :
11137 : }
11138 :
11139 : static inline int on_null_domain(struct rq *rq)
11140 : {
11141 : return unlikely(!rcu_dereference_sched(rq->sd));
11142 : }
11143 :
11144 : #ifdef CONFIG_NO_HZ_COMMON
11145 : /*
11146 : * idle load balancing details
11147 : * - When one of the busy CPUs notice that there may be an idle rebalancing
11148 : * needed, they will kick the idle load balancer, which then does idle
11149 : * load balancing for all the idle CPUs.
11150 : * - HK_TYPE_MISC CPUs are used for this task, because HK_TYPE_SCHED not set
11151 : * anywhere yet.
11152 : */
11153 :
11154 : static inline int find_new_ilb(void)
11155 : {
11156 : int ilb;
11157 : const struct cpumask *hk_mask;
11158 :
11159 : hk_mask = housekeeping_cpumask(HK_TYPE_MISC);
11160 :
11161 : for_each_cpu_and(ilb, nohz.idle_cpus_mask, hk_mask) {
11162 :
11163 : if (ilb == smp_processor_id())
11164 : continue;
11165 :
11166 : if (idle_cpu(ilb))
11167 : return ilb;
11168 : }
11169 :
11170 : return nr_cpu_ids;
11171 : }
11172 :
11173 : /*
11174 : * Kick a CPU to do the nohz balancing, if it is time for it. We pick any
11175 : * idle CPU in the HK_TYPE_MISC housekeeping set (if there is one).
11176 : */
11177 : static void kick_ilb(unsigned int flags)
11178 : {
11179 : int ilb_cpu;
11180 :
11181 : /*
11182 : * Increase nohz.next_balance only when if full ilb is triggered but
11183 : * not if we only update stats.
11184 : */
11185 : if (flags & NOHZ_BALANCE_KICK)
11186 : nohz.next_balance = jiffies+1;
11187 :
11188 : ilb_cpu = find_new_ilb();
11189 :
11190 : if (ilb_cpu >= nr_cpu_ids)
11191 : return;
11192 :
11193 : /*
11194 : * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
11195 : * the first flag owns it; cleared by nohz_csd_func().
11196 : */
11197 : flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu));
11198 : if (flags & NOHZ_KICK_MASK)
11199 : return;
11200 :
11201 : /*
11202 : * This way we generate an IPI on the target CPU which
11203 : * is idle. And the softirq performing nohz idle load balance
11204 : * will be run before returning from the IPI.
11205 : */
11206 : smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);
11207 : }
11208 :
11209 : /*
11210 : * Current decision point for kicking the idle load balancer in the presence
11211 : * of idle CPUs in the system.
11212 : */
11213 : static void nohz_balancer_kick(struct rq *rq)
11214 : {
11215 : unsigned long now = jiffies;
11216 : struct sched_domain_shared *sds;
11217 : struct sched_domain *sd;
11218 : int nr_busy, i, cpu = rq->cpu;
11219 : unsigned int flags = 0;
11220 :
11221 : if (unlikely(rq->idle_balance))
11222 : return;
11223 :
11224 : /*
11225 : * We may be recently in ticked or tickless idle mode. At the first
11226 : * busy tick after returning from idle, we will update the busy stats.
11227 : */
11228 : nohz_balance_exit_idle(rq);
11229 :
11230 : /*
11231 : * None are in tickless mode and hence no need for NOHZ idle load
11232 : * balancing.
11233 : */
11234 : if (likely(!atomic_read(&nohz.nr_cpus)))
11235 : return;
11236 :
11237 : if (READ_ONCE(nohz.has_blocked) &&
11238 : time_after(now, READ_ONCE(nohz.next_blocked)))
11239 : flags = NOHZ_STATS_KICK;
11240 :
11241 : if (time_before(now, nohz.next_balance))
11242 : goto out;
11243 :
11244 : if (rq->nr_running >= 2) {
11245 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11246 : goto out;
11247 : }
11248 :
11249 : rcu_read_lock();
11250 :
11251 : sd = rcu_dereference(rq->sd);
11252 : if (sd) {
11253 : /*
11254 : * If there's a CFS task and the current CPU has reduced
11255 : * capacity; kick the ILB to see if there's a better CPU to run
11256 : * on.
11257 : */
11258 : if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) {
11259 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11260 : goto unlock;
11261 : }
11262 : }
11263 :
11264 : sd = rcu_dereference(per_cpu(sd_asym_packing, cpu));
11265 : if (sd) {
11266 : /*
11267 : * When ASYM_PACKING; see if there's a more preferred CPU
11268 : * currently idle; in which case, kick the ILB to move tasks
11269 : * around.
11270 : */
11271 : for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
11272 : if (sched_asym_prefer(i, cpu)) {
11273 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11274 : goto unlock;
11275 : }
11276 : }
11277 : }
11278 :
11279 : sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu));
11280 : if (sd) {
11281 : /*
11282 : * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
11283 : * to run the misfit task on.
11284 : */
11285 : if (check_misfit_status(rq, sd)) {
11286 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11287 : goto unlock;
11288 : }
11289 :
11290 : /*
11291 : * For asymmetric systems, we do not want to nicely balance
11292 : * cache use, instead we want to embrace asymmetry and only
11293 : * ensure tasks have enough CPU capacity.
11294 : *
11295 : * Skip the LLC logic because it's not relevant in that case.
11296 : */
11297 : goto unlock;
11298 : }
11299 :
11300 : sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
11301 : if (sds) {
11302 : /*
11303 : * If there is an imbalance between LLC domains (IOW we could
11304 : * increase the overall cache use), we need some less-loaded LLC
11305 : * domain to pull some load. Likewise, we may need to spread
11306 : * load within the current LLC domain (e.g. packed SMT cores but
11307 : * other CPUs are idle). We can't really know from here how busy
11308 : * the others are - so just get a nohz balance going if it looks
11309 : * like this LLC domain has tasks we could move.
11310 : */
11311 : nr_busy = atomic_read(&sds->nr_busy_cpus);
11312 : if (nr_busy > 1) {
11313 : flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11314 : goto unlock;
11315 : }
11316 : }
11317 : unlock:
11318 : rcu_read_unlock();
11319 : out:
11320 : if (READ_ONCE(nohz.needs_update))
11321 : flags |= NOHZ_NEXT_KICK;
11322 :
11323 : if (flags)
11324 : kick_ilb(flags);
11325 : }
11326 :
11327 : static void set_cpu_sd_state_busy(int cpu)
11328 : {
11329 : struct sched_domain *sd;
11330 :
11331 : rcu_read_lock();
11332 : sd = rcu_dereference(per_cpu(sd_llc, cpu));
11333 :
11334 : if (!sd || !sd->nohz_idle)
11335 : goto unlock;
11336 : sd->nohz_idle = 0;
11337 :
11338 : atomic_inc(&sd->shared->nr_busy_cpus);
11339 : unlock:
11340 : rcu_read_unlock();
11341 : }
11342 :
11343 : void nohz_balance_exit_idle(struct rq *rq)
11344 : {
11345 : SCHED_WARN_ON(rq != this_rq());
11346 :
11347 : if (likely(!rq->nohz_tick_stopped))
11348 : return;
11349 :
11350 : rq->nohz_tick_stopped = 0;
11351 : cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask);
11352 : atomic_dec(&nohz.nr_cpus);
11353 :
11354 : set_cpu_sd_state_busy(rq->cpu);
11355 : }
11356 :
11357 : static void set_cpu_sd_state_idle(int cpu)
11358 : {
11359 : struct sched_domain *sd;
11360 :
11361 : rcu_read_lock();
11362 : sd = rcu_dereference(per_cpu(sd_llc, cpu));
11363 :
11364 : if (!sd || sd->nohz_idle)
11365 : goto unlock;
11366 : sd->nohz_idle = 1;
11367 :
11368 : atomic_dec(&sd->shared->nr_busy_cpus);
11369 : unlock:
11370 : rcu_read_unlock();
11371 : }
11372 :
11373 : /*
11374 : * This routine will record that the CPU is going idle with tick stopped.
11375 : * This info will be used in performing idle load balancing in the future.
11376 : */
11377 : void nohz_balance_enter_idle(int cpu)
11378 : {
11379 : struct rq *rq = cpu_rq(cpu);
11380 :
11381 : SCHED_WARN_ON(cpu != smp_processor_id());
11382 :
11383 : /* If this CPU is going down, then nothing needs to be done: */
11384 : if (!cpu_active(cpu))
11385 : return;
11386 :
11387 : /* Spare idle load balancing on CPUs that don't want to be disturbed: */
11388 : if (!housekeeping_cpu(cpu, HK_TYPE_SCHED))
11389 : return;
11390 :
11391 : /*
11392 : * Can be set safely without rq->lock held
11393 : * If a clear happens, it will have evaluated last additions because
11394 : * rq->lock is held during the check and the clear
11395 : */
11396 : rq->has_blocked_load = 1;
11397 :
11398 : /*
11399 : * The tick is still stopped but load could have been added in the
11400 : * meantime. We set the nohz.has_blocked flag to trig a check of the
11401 : * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear
11402 : * of nohz.has_blocked can only happen after checking the new load
11403 : */
11404 : if (rq->nohz_tick_stopped)
11405 : goto out;
11406 :
11407 : /* If we're a completely isolated CPU, we don't play: */
11408 : if (on_null_domain(rq))
11409 : return;
11410 :
11411 : rq->nohz_tick_stopped = 1;
11412 :
11413 : cpumask_set_cpu(cpu, nohz.idle_cpus_mask);
11414 : atomic_inc(&nohz.nr_cpus);
11415 :
11416 : /*
11417 : * Ensures that if nohz_idle_balance() fails to observe our
11418 : * @idle_cpus_mask store, it must observe the @has_blocked
11419 : * and @needs_update stores.
11420 : */
11421 : smp_mb__after_atomic();
11422 :
11423 : set_cpu_sd_state_idle(cpu);
11424 :
11425 : WRITE_ONCE(nohz.needs_update, 1);
11426 : out:
11427 : /*
11428 : * Each time a cpu enter idle, we assume that it has blocked load and
11429 : * enable the periodic update of the load of idle cpus
11430 : */
11431 : WRITE_ONCE(nohz.has_blocked, 1);
11432 : }
11433 :
11434 : static bool update_nohz_stats(struct rq *rq)
11435 : {
11436 : unsigned int cpu = rq->cpu;
11437 :
11438 : if (!rq->has_blocked_load)
11439 : return false;
11440 :
11441 : if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask))
11442 : return false;
11443 :
11444 : if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick)))
11445 : return true;
11446 :
11447 : update_blocked_averages(cpu);
11448 :
11449 : return rq->has_blocked_load;
11450 : }
11451 :
11452 : /*
11453 : * Internal function that runs load balance for all idle cpus. The load balance
11454 : * can be a simple update of blocked load or a complete load balance with
11455 : * tasks movement depending of flags.
11456 : */
11457 : static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)
11458 : {
11459 : /* Earliest time when we have to do rebalance again */
11460 : unsigned long now = jiffies;
11461 : unsigned long next_balance = now + 60*HZ;
11462 : bool has_blocked_load = false;
11463 : int update_next_balance = 0;
11464 : int this_cpu = this_rq->cpu;
11465 : int balance_cpu;
11466 : struct rq *rq;
11467 :
11468 : SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
11469 :
11470 : /*
11471 : * We assume there will be no idle load after this update and clear
11472 : * the has_blocked flag. If a cpu enters idle in the mean time, it will
11473 : * set the has_blocked flag and trigger another update of idle load.
11474 : * Because a cpu that becomes idle, is added to idle_cpus_mask before
11475 : * setting the flag, we are sure to not clear the state and not
11476 : * check the load of an idle cpu.
11477 : *
11478 : * Same applies to idle_cpus_mask vs needs_update.
11479 : */
11480 : if (flags & NOHZ_STATS_KICK)
11481 : WRITE_ONCE(nohz.has_blocked, 0);
11482 : if (flags & NOHZ_NEXT_KICK)
11483 : WRITE_ONCE(nohz.needs_update, 0);
11484 :
11485 : /*
11486 : * Ensures that if we miss the CPU, we must see the has_blocked
11487 : * store from nohz_balance_enter_idle().
11488 : */
11489 : smp_mb();
11490 :
11491 : /*
11492 : * Start with the next CPU after this_cpu so we will end with this_cpu and let a
11493 : * chance for other idle cpu to pull load.
11494 : */
11495 : for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
11496 : if (!idle_cpu(balance_cpu))
11497 : continue;
11498 :
11499 : /*
11500 : * If this CPU gets work to do, stop the load balancing
11501 : * work being done for other CPUs. Next load
11502 : * balancing owner will pick it up.
11503 : */
11504 : if (need_resched()) {
11505 : if (flags & NOHZ_STATS_KICK)
11506 : has_blocked_load = true;
11507 : if (flags & NOHZ_NEXT_KICK)
11508 : WRITE_ONCE(nohz.needs_update, 1);
11509 : goto abort;
11510 : }
11511 :
11512 : rq = cpu_rq(balance_cpu);
11513 :
11514 : if (flags & NOHZ_STATS_KICK)
11515 : has_blocked_load |= update_nohz_stats(rq);
11516 :
11517 : /*
11518 : * If time for next balance is due,
11519 : * do the balance.
11520 : */
11521 : if (time_after_eq(jiffies, rq->next_balance)) {
11522 : struct rq_flags rf;
11523 :
11524 : rq_lock_irqsave(rq, &rf);
11525 : update_rq_clock(rq);
11526 : rq_unlock_irqrestore(rq, &rf);
11527 :
11528 : if (flags & NOHZ_BALANCE_KICK)
11529 : rebalance_domains(rq, CPU_IDLE);
11530 : }
11531 :
11532 : if (time_after(next_balance, rq->next_balance)) {
11533 : next_balance = rq->next_balance;
11534 : update_next_balance = 1;
11535 : }
11536 : }
11537 :
11538 : /*
11539 : * next_balance will be updated only when there is a need.
11540 : * When the CPU is attached to null domain for ex, it will not be
11541 : * updated.
11542 : */
11543 : if (likely(update_next_balance))
11544 : nohz.next_balance = next_balance;
11545 :
11546 : if (flags & NOHZ_STATS_KICK)
11547 : WRITE_ONCE(nohz.next_blocked,
11548 : now + msecs_to_jiffies(LOAD_AVG_PERIOD));
11549 :
11550 : abort:
11551 : /* There is still blocked load, enable periodic update */
11552 : if (has_blocked_load)
11553 : WRITE_ONCE(nohz.has_blocked, 1);
11554 : }
11555 :
11556 : /*
11557 : * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
11558 : * rebalancing for all the cpus for whom scheduler ticks are stopped.
11559 : */
11560 : static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
11561 : {
11562 : unsigned int flags = this_rq->nohz_idle_balance;
11563 :
11564 : if (!flags)
11565 : return false;
11566 :
11567 : this_rq->nohz_idle_balance = 0;
11568 :
11569 : if (idle != CPU_IDLE)
11570 : return false;
11571 :
11572 : _nohz_idle_balance(this_rq, flags);
11573 :
11574 : return true;
11575 : }
11576 :
11577 : /*
11578 : * Check if we need to run the ILB for updating blocked load before entering
11579 : * idle state.
11580 : */
11581 : void nohz_run_idle_balance(int cpu)
11582 : {
11583 : unsigned int flags;
11584 :
11585 : flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu));
11586 :
11587 : /*
11588 : * Update the blocked load only if no SCHED_SOFTIRQ is about to happen
11589 : * (ie NOHZ_STATS_KICK set) and will do the same.
11590 : */
11591 : if ((flags == NOHZ_NEWILB_KICK) && !need_resched())
11592 : _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK);
11593 : }
11594 :
11595 : static void nohz_newidle_balance(struct rq *this_rq)
11596 : {
11597 : int this_cpu = this_rq->cpu;
11598 :
11599 : /*
11600 : * This CPU doesn't want to be disturbed by scheduler
11601 : * housekeeping
11602 : */
11603 : if (!housekeeping_cpu(this_cpu, HK_TYPE_SCHED))
11604 : return;
11605 :
11606 : /* Will wake up very soon. No time for doing anything else*/
11607 : if (this_rq->avg_idle < sysctl_sched_migration_cost)
11608 : return;
11609 :
11610 : /* Don't need to update blocked load of idle CPUs*/
11611 : if (!READ_ONCE(nohz.has_blocked) ||
11612 : time_before(jiffies, READ_ONCE(nohz.next_blocked)))
11613 : return;
11614 :
11615 : /*
11616 : * Set the need to trigger ILB in order to update blocked load
11617 : * before entering idle state.
11618 : */
11619 : atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu));
11620 : }
11621 :
11622 : #else /* !CONFIG_NO_HZ_COMMON */
11623 : static inline void nohz_balancer_kick(struct rq *rq) { }
11624 :
11625 : static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
11626 : {
11627 : return false;
11628 : }
11629 :
11630 : static inline void nohz_newidle_balance(struct rq *this_rq) { }
11631 : #endif /* CONFIG_NO_HZ_COMMON */
11632 :
11633 : /*
11634 : * newidle_balance is called by schedule() if this_cpu is about to become
11635 : * idle. Attempts to pull tasks from other CPUs.
11636 : *
11637 : * Returns:
11638 : * < 0 - we released the lock and there are !fair tasks present
11639 : * 0 - failed, no new tasks
11640 : * > 0 - success, new (fair) tasks present
11641 : */
11642 : static int newidle_balance(struct rq *this_rq, struct rq_flags *rf)
11643 : {
11644 : unsigned long next_balance = jiffies + HZ;
11645 : int this_cpu = this_rq->cpu;
11646 : u64 t0, t1, curr_cost = 0;
11647 : struct sched_domain *sd;
11648 : int pulled_task = 0;
11649 :
11650 : update_misfit_status(NULL, this_rq);
11651 :
11652 : /*
11653 : * There is a task waiting to run. No need to search for one.
11654 : * Return 0; the task will be enqueued when switching to idle.
11655 : */
11656 : if (this_rq->ttwu_pending)
11657 : return 0;
11658 :
11659 : /*
11660 : * We must set idle_stamp _before_ calling idle_balance(), such that we
11661 : * measure the duration of idle_balance() as idle time.
11662 : */
11663 : this_rq->idle_stamp = rq_clock(this_rq);
11664 :
11665 : /*
11666 : * Do not pull tasks towards !active CPUs...
11667 : */
11668 : if (!cpu_active(this_cpu))
11669 : return 0;
11670 :
11671 : /*
11672 : * This is OK, because current is on_cpu, which avoids it being picked
11673 : * for load-balance and preemption/IRQs are still disabled avoiding
11674 : * further scheduler activity on it and we're being very careful to
11675 : * re-start the picking loop.
11676 : */
11677 : rq_unpin_lock(this_rq, rf);
11678 :
11679 : rcu_read_lock();
11680 : sd = rcu_dereference_check_sched_domain(this_rq->sd);
11681 :
11682 : if (!READ_ONCE(this_rq->rd->overload) ||
11683 : (sd && this_rq->avg_idle < sd->max_newidle_lb_cost)) {
11684 :
11685 : if (sd)
11686 : update_next_balance(sd, &next_balance);
11687 : rcu_read_unlock();
11688 :
11689 : goto out;
11690 : }
11691 : rcu_read_unlock();
11692 :
11693 : raw_spin_rq_unlock(this_rq);
11694 :
11695 : t0 = sched_clock_cpu(this_cpu);
11696 : update_blocked_averages(this_cpu);
11697 :
11698 : rcu_read_lock();
11699 : for_each_domain(this_cpu, sd) {
11700 : int continue_balancing = 1;
11701 : u64 domain_cost;
11702 :
11703 : update_next_balance(sd, &next_balance);
11704 :
11705 : if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)
11706 : break;
11707 :
11708 : if (sd->flags & SD_BALANCE_NEWIDLE) {
11709 :
11710 : pulled_task = load_balance(this_cpu, this_rq,
11711 : sd, CPU_NEWLY_IDLE,
11712 : &continue_balancing);
11713 :
11714 : t1 = sched_clock_cpu(this_cpu);
11715 : domain_cost = t1 - t0;
11716 : update_newidle_cost(sd, domain_cost);
11717 :
11718 : curr_cost += domain_cost;
11719 : t0 = t1;
11720 : }
11721 :
11722 : /*
11723 : * Stop searching for tasks to pull if there are
11724 : * now runnable tasks on this rq.
11725 : */
11726 : if (pulled_task || this_rq->nr_running > 0 ||
11727 : this_rq->ttwu_pending)
11728 : break;
11729 : }
11730 : rcu_read_unlock();
11731 :
11732 : raw_spin_rq_lock(this_rq);
11733 :
11734 : if (curr_cost > this_rq->max_idle_balance_cost)
11735 : this_rq->max_idle_balance_cost = curr_cost;
11736 :
11737 : /*
11738 : * While browsing the domains, we released the rq lock, a task could
11739 : * have been enqueued in the meantime. Since we're not going idle,
11740 : * pretend we pulled a task.
11741 : */
11742 : if (this_rq->cfs.h_nr_running && !pulled_task)
11743 : pulled_task = 1;
11744 :
11745 : /* Is there a task of a high priority class? */
11746 : if (this_rq->nr_running != this_rq->cfs.h_nr_running)
11747 : pulled_task = -1;
11748 :
11749 : out:
11750 : /* Move the next balance forward */
11751 : if (time_after(this_rq->next_balance, next_balance))
11752 : this_rq->next_balance = next_balance;
11753 :
11754 : if (pulled_task)
11755 : this_rq->idle_stamp = 0;
11756 : else
11757 : nohz_newidle_balance(this_rq);
11758 :
11759 : rq_repin_lock(this_rq, rf);
11760 :
11761 : return pulled_task;
11762 : }
11763 :
11764 : /*
11765 : * run_rebalance_domains is triggered when needed from the scheduler tick.
11766 : * Also triggered for nohz idle balancing (with nohz_balancing_kick set).
11767 : */
11768 : static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
11769 : {
11770 : struct rq *this_rq = this_rq();
11771 : enum cpu_idle_type idle = this_rq->idle_balance ?
11772 : CPU_IDLE : CPU_NOT_IDLE;
11773 :
11774 : /*
11775 : * If this CPU has a pending nohz_balance_kick, then do the
11776 : * balancing on behalf of the other idle CPUs whose ticks are
11777 : * stopped. Do nohz_idle_balance *before* rebalance_domains to
11778 : * give the idle CPUs a chance to load balance. Else we may
11779 : * load balance only within the local sched_domain hierarchy
11780 : * and abort nohz_idle_balance altogether if we pull some load.
11781 : */
11782 : if (nohz_idle_balance(this_rq, idle))
11783 : return;
11784 :
11785 : /* normal load balance */
11786 : update_blocked_averages(this_rq->cpu);
11787 : rebalance_domains(this_rq, idle);
11788 : }
11789 :
11790 : /*
11791 : * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
11792 : */
11793 : void trigger_load_balance(struct rq *rq)
11794 : {
11795 : /*
11796 : * Don't need to rebalance while attached to NULL domain or
11797 : * runqueue CPU is not active
11798 : */
11799 : if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq))))
11800 : return;
11801 :
11802 : if (time_after_eq(jiffies, rq->next_balance))
11803 : raise_softirq(SCHED_SOFTIRQ);
11804 :
11805 : nohz_balancer_kick(rq);
11806 : }
11807 :
11808 : static void rq_online_fair(struct rq *rq)
11809 : {
11810 : update_sysctl();
11811 :
11812 : update_runtime_enabled(rq);
11813 : }
11814 :
11815 : static void rq_offline_fair(struct rq *rq)
11816 : {
11817 : update_sysctl();
11818 :
11819 : /* Ensure any throttled groups are reachable by pick_next_task */
11820 : unthrottle_offline_cfs_rqs(rq);
11821 : }
11822 :
11823 : #endif /* CONFIG_SMP */
11824 :
11825 : #ifdef CONFIG_SCHED_CORE
11826 : static inline bool
11827 : __entity_slice_used(struct sched_entity *se, int min_nr_tasks)
11828 : {
11829 : u64 slice = sched_slice(cfs_rq_of(se), se);
11830 : u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime;
11831 :
11832 : return (rtime * min_nr_tasks > slice);
11833 : }
11834 :
11835 : #define MIN_NR_TASKS_DURING_FORCEIDLE 2
11836 : static inline void task_tick_core(struct rq *rq, struct task_struct *curr)
11837 : {
11838 : if (!sched_core_enabled(rq))
11839 : return;
11840 :
11841 : /*
11842 : * If runqueue has only one task which used up its slice and
11843 : * if the sibling is forced idle, then trigger schedule to
11844 : * give forced idle task a chance.
11845 : *
11846 : * sched_slice() considers only this active rq and it gets the
11847 : * whole slice. But during force idle, we have siblings acting
11848 : * like a single runqueue and hence we need to consider runnable
11849 : * tasks on this CPU and the forced idle CPU. Ideally, we should
11850 : * go through the forced idle rq, but that would be a perf hit.
11851 : * We can assume that the forced idle CPU has at least
11852 : * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check
11853 : * if we need to give up the CPU.
11854 : */
11855 : if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 &&
11856 : __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))
11857 : resched_curr(rq);
11858 : }
11859 :
11860 : /*
11861 : * se_fi_update - Update the cfs_rq->min_vruntime_fi in a CFS hierarchy if needed.
11862 : */
11863 : static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq,
11864 : bool forceidle)
11865 : {
11866 : for_each_sched_entity(se) {
11867 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
11868 :
11869 : if (forceidle) {
11870 : if (cfs_rq->forceidle_seq == fi_seq)
11871 : break;
11872 : cfs_rq->forceidle_seq = fi_seq;
11873 : }
11874 :
11875 : cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime;
11876 : }
11877 : }
11878 :
11879 : void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi)
11880 : {
11881 : struct sched_entity *se = &p->se;
11882 :
11883 : if (p->sched_class != &fair_sched_class)
11884 : return;
11885 :
11886 : se_fi_update(se, rq->core->core_forceidle_seq, in_fi);
11887 : }
11888 :
11889 : bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b,
11890 : bool in_fi)
11891 : {
11892 : struct rq *rq = task_rq(a);
11893 : const struct sched_entity *sea = &a->se;
11894 : const struct sched_entity *seb = &b->se;
11895 : struct cfs_rq *cfs_rqa;
11896 : struct cfs_rq *cfs_rqb;
11897 : s64 delta;
11898 :
11899 : SCHED_WARN_ON(task_rq(b)->core != rq->core);
11900 :
11901 : #ifdef CONFIG_FAIR_GROUP_SCHED
11902 : /*
11903 : * Find an se in the hierarchy for tasks a and b, such that the se's
11904 : * are immediate siblings.
11905 : */
11906 : while (sea->cfs_rq->tg != seb->cfs_rq->tg) {
11907 : int sea_depth = sea->depth;
11908 : int seb_depth = seb->depth;
11909 :
11910 : if (sea_depth >= seb_depth)
11911 : sea = parent_entity(sea);
11912 : if (sea_depth <= seb_depth)
11913 : seb = parent_entity(seb);
11914 : }
11915 :
11916 : se_fi_update(sea, rq->core->core_forceidle_seq, in_fi);
11917 : se_fi_update(seb, rq->core->core_forceidle_seq, in_fi);
11918 :
11919 : cfs_rqa = sea->cfs_rq;
11920 : cfs_rqb = seb->cfs_rq;
11921 : #else
11922 : cfs_rqa = &task_rq(a)->cfs;
11923 : cfs_rqb = &task_rq(b)->cfs;
11924 : #endif
11925 :
11926 : /*
11927 : * Find delta after normalizing se's vruntime with its cfs_rq's
11928 : * min_vruntime_fi, which would have been updated in prior calls
11929 : * to se_fi_update().
11930 : */
11931 : delta = (s64)(sea->vruntime - seb->vruntime) +
11932 : (s64)(cfs_rqb->min_vruntime_fi - cfs_rqa->min_vruntime_fi);
11933 :
11934 : return delta > 0;
11935 : }
11936 : #else
11937 : static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}
11938 : #endif
11939 :
11940 : /*
11941 : * scheduler tick hitting a task of our scheduling class.
11942 : *
11943 : * NOTE: This function can be called remotely by the tick offload that
11944 : * goes along full dynticks. Therefore no local assumption can be made
11945 : * and everything must be accessed through the @rq and @curr passed in
11946 : * parameters.
11947 : */
11948 2719 : static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
11949 : {
11950 : struct cfs_rq *cfs_rq;
11951 2719 : struct sched_entity *se = &curr->se;
11952 :
11953 5438 : for_each_sched_entity(se) {
11954 5438 : cfs_rq = cfs_rq_of(se);
11955 2719 : entity_tick(cfs_rq, se, queued);
11956 : }
11957 :
11958 2719 : if (static_branch_unlikely(&sched_numa_balancing))
11959 : task_tick_numa(rq, curr);
11960 :
11961 2719 : update_misfit_status(curr, rq);
11962 2719 : update_overutilized_status(task_rq(curr));
11963 :
11964 2719 : task_tick_core(rq, curr);
11965 2719 : }
11966 :
11967 : /*
11968 : * called on fork with the child task as argument from the parent's context
11969 : * - child not yet on the tasklist
11970 : * - preemption disabled
11971 : */
11972 340 : static void task_fork_fair(struct task_struct *p)
11973 : {
11974 : struct cfs_rq *cfs_rq;
11975 340 : struct sched_entity *se = &p->se, *curr;
11976 340 : struct rq *rq = this_rq();
11977 : struct rq_flags rf;
11978 :
11979 340 : rq_lock(rq, &rf);
11980 340 : update_rq_clock(rq);
11981 :
11982 680 : cfs_rq = task_cfs_rq(current);
11983 340 : curr = cfs_rq->curr;
11984 340 : if (curr) {
11985 338 : update_curr(cfs_rq);
11986 338 : se->vruntime = curr->vruntime;
11987 : }
11988 340 : place_entity(cfs_rq, se, 1);
11989 :
11990 340 : if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) {
11991 : /*
11992 : * Upon rescheduling, sched_class::put_prev_task() will place
11993 : * 'current' within the tree based on its new key value.
11994 : */
11995 0 : swap(curr->vruntime, se->vruntime);
11996 0 : resched_curr(rq);
11997 : }
11998 :
11999 340 : se->vruntime -= cfs_rq->min_vruntime;
12000 340 : rq_unlock(rq, &rf);
12001 340 : }
12002 :
12003 : /*
12004 : * Priority of the task has changed. Check to see if we preempt
12005 : * the current task.
12006 : */
12007 : static void
12008 5 : prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio)
12009 : {
12010 5 : if (!task_on_rq_queued(p))
12011 : return;
12012 :
12013 4 : if (rq->cfs.nr_running == 1)
12014 : return;
12015 :
12016 : /*
12017 : * Reschedule if we are currently running on this runqueue and
12018 : * our priority decreased, or if we are not currently running on
12019 : * this runqueue and our priority is higher than the current's
12020 : */
12021 4 : if (task_current(rq, p)) {
12022 4 : if (p->prio > oldprio)
12023 0 : resched_curr(rq);
12024 : } else
12025 0 : check_preempt_curr(rq, p, 0);
12026 : }
12027 :
12028 : static inline bool vruntime_normalized(struct task_struct *p)
12029 : {
12030 0 : struct sched_entity *se = &p->se;
12031 :
12032 : /*
12033 : * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases,
12034 : * the dequeue_entity(.flags=0) will already have normalized the
12035 : * vruntime.
12036 : */
12037 0 : if (p->on_rq)
12038 : return true;
12039 :
12040 : /*
12041 : * When !on_rq, vruntime of the task has usually NOT been normalized.
12042 : * But there are some cases where it has already been normalized:
12043 : *
12044 : * - A forked child which is waiting for being woken up by
12045 : * wake_up_new_task().
12046 : * - A task which has been woken up by try_to_wake_up() and
12047 : * waiting for actually being woken up by sched_ttwu_pending().
12048 : */
12049 0 : if (!se->sum_exec_runtime ||
12050 0 : (READ_ONCE(p->__state) == TASK_WAKING && p->sched_remote_wakeup))
12051 : return true;
12052 :
12053 : return false;
12054 : }
12055 :
12056 : #ifdef CONFIG_FAIR_GROUP_SCHED
12057 : /*
12058 : * Propagate the changes of the sched_entity across the tg tree to make it
12059 : * visible to the root
12060 : */
12061 : static void propagate_entity_cfs_rq(struct sched_entity *se)
12062 : {
12063 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
12064 :
12065 : if (cfs_rq_throttled(cfs_rq))
12066 : return;
12067 :
12068 : if (!throttled_hierarchy(cfs_rq))
12069 : list_add_leaf_cfs_rq(cfs_rq);
12070 :
12071 : /* Start to propagate at parent */
12072 : se = se->parent;
12073 :
12074 : for_each_sched_entity(se) {
12075 : cfs_rq = cfs_rq_of(se);
12076 :
12077 : update_load_avg(cfs_rq, se, UPDATE_TG);
12078 :
12079 : if (cfs_rq_throttled(cfs_rq))
12080 : break;
12081 :
12082 : if (!throttled_hierarchy(cfs_rq))
12083 : list_add_leaf_cfs_rq(cfs_rq);
12084 : }
12085 : }
12086 : #else
12087 : static void propagate_entity_cfs_rq(struct sched_entity *se) { }
12088 : #endif
12089 :
12090 : static void detach_entity_cfs_rq(struct sched_entity *se)
12091 : {
12092 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
12093 :
12094 : #ifdef CONFIG_SMP
12095 : /*
12096 : * In case the task sched_avg hasn't been attached:
12097 : * - A forked task which hasn't been woken up by wake_up_new_task().
12098 : * - A task which has been woken up by try_to_wake_up() but is
12099 : * waiting for actually being woken up by sched_ttwu_pending().
12100 : */
12101 : if (!se->avg.last_update_time)
12102 : return;
12103 : #endif
12104 :
12105 : /* Catch up with the cfs_rq and remove our load when we leave */
12106 0 : update_load_avg(cfs_rq, se, 0);
12107 0 : detach_entity_load_avg(cfs_rq, se);
12108 : update_tg_load_avg(cfs_rq);
12109 0 : propagate_entity_cfs_rq(se);
12110 : }
12111 :
12112 : static void attach_entity_cfs_rq(struct sched_entity *se)
12113 : {
12114 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
12115 :
12116 : /* Synchronize entity with its cfs_rq */
12117 0 : update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);
12118 0 : attach_entity_load_avg(cfs_rq, se);
12119 : update_tg_load_avg(cfs_rq);
12120 0 : propagate_entity_cfs_rq(se);
12121 : }
12122 :
12123 0 : static void detach_task_cfs_rq(struct task_struct *p)
12124 : {
12125 0 : struct sched_entity *se = &p->se;
12126 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
12127 :
12128 0 : if (!vruntime_normalized(p)) {
12129 : /*
12130 : * Fix up our vruntime so that the current sleep doesn't
12131 : * cause 'unlimited' sleep bonus.
12132 : */
12133 0 : place_entity(cfs_rq, se, 0);
12134 0 : se->vruntime -= cfs_rq->min_vruntime;
12135 : }
12136 :
12137 0 : detach_entity_cfs_rq(se);
12138 0 : }
12139 :
12140 : static void attach_task_cfs_rq(struct task_struct *p)
12141 : {
12142 0 : struct sched_entity *se = &p->se;
12143 0 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
12144 :
12145 0 : attach_entity_cfs_rq(se);
12146 :
12147 0 : if (!vruntime_normalized(p))
12148 0 : se->vruntime += cfs_rq->min_vruntime;
12149 : }
12150 :
12151 0 : static void switched_from_fair(struct rq *rq, struct task_struct *p)
12152 : {
12153 0 : detach_task_cfs_rq(p);
12154 0 : }
12155 :
12156 0 : static void switched_to_fair(struct rq *rq, struct task_struct *p)
12157 : {
12158 0 : attach_task_cfs_rq(p);
12159 :
12160 0 : if (task_on_rq_queued(p)) {
12161 : /*
12162 : * We were most likely switched from sched_rt, so
12163 : * kick off the schedule if running, otherwise just see
12164 : * if we can still preempt the current task.
12165 : */
12166 0 : if (task_current(rq, p))
12167 0 : resched_curr(rq);
12168 : else
12169 0 : check_preempt_curr(rq, p, 0);
12170 : }
12171 0 : }
12172 :
12173 : /* Account for a task changing its policy or group.
12174 : *
12175 : * This routine is mostly called to set cfs_rq->curr field when a task
12176 : * migrates between groups/classes.
12177 : */
12178 4 : static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
12179 : {
12180 4 : struct sched_entity *se = &p->se;
12181 :
12182 : #ifdef CONFIG_SMP
12183 : if (task_on_rq_queued(p)) {
12184 : /*
12185 : * Move the next running task to the front of the list, so our
12186 : * cfs_tasks list becomes MRU one.
12187 : */
12188 : list_move(&se->group_node, &rq->cfs_tasks);
12189 : }
12190 : #endif
12191 :
12192 8 : for_each_sched_entity(se) {
12193 8 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
12194 :
12195 4 : set_next_entity(cfs_rq, se);
12196 : /* ensure bandwidth has been allocated on our new cfs_rq */
12197 4 : account_cfs_rq_runtime(cfs_rq, 0);
12198 : }
12199 4 : }
12200 :
12201 1 : void init_cfs_rq(struct cfs_rq *cfs_rq)
12202 : {
12203 1 : cfs_rq->tasks_timeline = RB_ROOT_CACHED;
12204 1 : u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20)));
12205 : #ifdef CONFIG_SMP
12206 : raw_spin_lock_init(&cfs_rq->removed.lock);
12207 : #endif
12208 1 : }
12209 :
12210 : #ifdef CONFIG_FAIR_GROUP_SCHED
12211 : static void task_change_group_fair(struct task_struct *p)
12212 : {
12213 : /*
12214 : * We couldn't detach or attach a forked task which
12215 : * hasn't been woken up by wake_up_new_task().
12216 : */
12217 : if (READ_ONCE(p->__state) == TASK_NEW)
12218 : return;
12219 :
12220 : detach_task_cfs_rq(p);
12221 :
12222 : #ifdef CONFIG_SMP
12223 : /* Tell se's cfs_rq has been changed -- migrated */
12224 : p->se.avg.last_update_time = 0;
12225 : #endif
12226 : set_task_rq(p, task_cpu(p));
12227 : attach_task_cfs_rq(p);
12228 : }
12229 :
12230 : void free_fair_sched_group(struct task_group *tg)
12231 : {
12232 : int i;
12233 :
12234 : for_each_possible_cpu(i) {
12235 : if (tg->cfs_rq)
12236 : kfree(tg->cfs_rq[i]);
12237 : if (tg->se)
12238 : kfree(tg->se[i]);
12239 : }
12240 :
12241 : kfree(tg->cfs_rq);
12242 : kfree(tg->se);
12243 : }
12244 :
12245 : int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
12246 : {
12247 : struct sched_entity *se;
12248 : struct cfs_rq *cfs_rq;
12249 : int i;
12250 :
12251 : tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
12252 : if (!tg->cfs_rq)
12253 : goto err;
12254 : tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL);
12255 : if (!tg->se)
12256 : goto err;
12257 :
12258 : tg->shares = NICE_0_LOAD;
12259 :
12260 : init_cfs_bandwidth(tg_cfs_bandwidth(tg));
12261 :
12262 : for_each_possible_cpu(i) {
12263 : cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
12264 : GFP_KERNEL, cpu_to_node(i));
12265 : if (!cfs_rq)
12266 : goto err;
12267 :
12268 : se = kzalloc_node(sizeof(struct sched_entity_stats),
12269 : GFP_KERNEL, cpu_to_node(i));
12270 : if (!se)
12271 : goto err_free_rq;
12272 :
12273 : init_cfs_rq(cfs_rq);
12274 : init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
12275 : init_entity_runnable_average(se);
12276 : }
12277 :
12278 : return 1;
12279 :
12280 : err_free_rq:
12281 : kfree(cfs_rq);
12282 : err:
12283 : return 0;
12284 : }
12285 :
12286 : void online_fair_sched_group(struct task_group *tg)
12287 : {
12288 : struct sched_entity *se;
12289 : struct rq_flags rf;
12290 : struct rq *rq;
12291 : int i;
12292 :
12293 : for_each_possible_cpu(i) {
12294 : rq = cpu_rq(i);
12295 : se = tg->se[i];
12296 : rq_lock_irq(rq, &rf);
12297 : update_rq_clock(rq);
12298 : attach_entity_cfs_rq(se);
12299 : sync_throttle(tg, i);
12300 : rq_unlock_irq(rq, &rf);
12301 : }
12302 : }
12303 :
12304 : void unregister_fair_sched_group(struct task_group *tg)
12305 : {
12306 : unsigned long flags;
12307 : struct rq *rq;
12308 : int cpu;
12309 :
12310 : destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
12311 :
12312 : for_each_possible_cpu(cpu) {
12313 : if (tg->se[cpu])
12314 : remove_entity_load_avg(tg->se[cpu]);
12315 :
12316 : /*
12317 : * Only empty task groups can be destroyed; so we can speculatively
12318 : * check on_list without danger of it being re-added.
12319 : */
12320 : if (!tg->cfs_rq[cpu]->on_list)
12321 : continue;
12322 :
12323 : rq = cpu_rq(cpu);
12324 :
12325 : raw_spin_rq_lock_irqsave(rq, flags);
12326 : list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
12327 : raw_spin_rq_unlock_irqrestore(rq, flags);
12328 : }
12329 : }
12330 :
12331 : void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
12332 : struct sched_entity *se, int cpu,
12333 : struct sched_entity *parent)
12334 : {
12335 : struct rq *rq = cpu_rq(cpu);
12336 :
12337 : cfs_rq->tg = tg;
12338 : cfs_rq->rq = rq;
12339 : init_cfs_rq_runtime(cfs_rq);
12340 :
12341 : tg->cfs_rq[cpu] = cfs_rq;
12342 : tg->se[cpu] = se;
12343 :
12344 : /* se could be NULL for root_task_group */
12345 : if (!se)
12346 : return;
12347 :
12348 : if (!parent) {
12349 : se->cfs_rq = &rq->cfs;
12350 : se->depth = 0;
12351 : } else {
12352 : se->cfs_rq = parent->my_q;
12353 : se->depth = parent->depth + 1;
12354 : }
12355 :
12356 : se->my_q = cfs_rq;
12357 : /* guarantee group entities always have weight */
12358 : update_load_set(&se->load, NICE_0_LOAD);
12359 : se->parent = parent;
12360 : }
12361 :
12362 : static DEFINE_MUTEX(shares_mutex);
12363 :
12364 : static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
12365 : {
12366 : int i;
12367 :
12368 : lockdep_assert_held(&shares_mutex);
12369 :
12370 : /*
12371 : * We can't change the weight of the root cgroup.
12372 : */
12373 : if (!tg->se[0])
12374 : return -EINVAL;
12375 :
12376 : shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
12377 :
12378 : if (tg->shares == shares)
12379 : return 0;
12380 :
12381 : tg->shares = shares;
12382 : for_each_possible_cpu(i) {
12383 : struct rq *rq = cpu_rq(i);
12384 : struct sched_entity *se = tg->se[i];
12385 : struct rq_flags rf;
12386 :
12387 : /* Propagate contribution to hierarchy */
12388 : rq_lock_irqsave(rq, &rf);
12389 : update_rq_clock(rq);
12390 : for_each_sched_entity(se) {
12391 : update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
12392 : update_cfs_group(se);
12393 : }
12394 : rq_unlock_irqrestore(rq, &rf);
12395 : }
12396 :
12397 : return 0;
12398 : }
12399 :
12400 : int sched_group_set_shares(struct task_group *tg, unsigned long shares)
12401 : {
12402 : int ret;
12403 :
12404 : mutex_lock(&shares_mutex);
12405 : if (tg_is_idle(tg))
12406 : ret = -EINVAL;
12407 : else
12408 : ret = __sched_group_set_shares(tg, shares);
12409 : mutex_unlock(&shares_mutex);
12410 :
12411 : return ret;
12412 : }
12413 :
12414 : int sched_group_set_idle(struct task_group *tg, long idle)
12415 : {
12416 : int i;
12417 :
12418 : if (tg == &root_task_group)
12419 : return -EINVAL;
12420 :
12421 : if (idle < 0 || idle > 1)
12422 : return -EINVAL;
12423 :
12424 : mutex_lock(&shares_mutex);
12425 :
12426 : if (tg->idle == idle) {
12427 : mutex_unlock(&shares_mutex);
12428 : return 0;
12429 : }
12430 :
12431 : tg->idle = idle;
12432 :
12433 : for_each_possible_cpu(i) {
12434 : struct rq *rq = cpu_rq(i);
12435 : struct sched_entity *se = tg->se[i];
12436 : struct cfs_rq *parent_cfs_rq, *grp_cfs_rq = tg->cfs_rq[i];
12437 : bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
12438 : long idle_task_delta;
12439 : struct rq_flags rf;
12440 :
12441 : rq_lock_irqsave(rq, &rf);
12442 :
12443 : grp_cfs_rq->idle = idle;
12444 : if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq)))
12445 : goto next_cpu;
12446 :
12447 : if (se->on_rq) {
12448 : parent_cfs_rq = cfs_rq_of(se);
12449 : if (cfs_rq_is_idle(grp_cfs_rq))
12450 : parent_cfs_rq->idle_nr_running++;
12451 : else
12452 : parent_cfs_rq->idle_nr_running--;
12453 : }
12454 :
12455 : idle_task_delta = grp_cfs_rq->h_nr_running -
12456 : grp_cfs_rq->idle_h_nr_running;
12457 : if (!cfs_rq_is_idle(grp_cfs_rq))
12458 : idle_task_delta *= -1;
12459 :
12460 : for_each_sched_entity(se) {
12461 : struct cfs_rq *cfs_rq = cfs_rq_of(se);
12462 :
12463 : if (!se->on_rq)
12464 : break;
12465 :
12466 : cfs_rq->idle_h_nr_running += idle_task_delta;
12467 :
12468 : /* Already accounted at parent level and above. */
12469 : if (cfs_rq_is_idle(cfs_rq))
12470 : break;
12471 : }
12472 :
12473 : next_cpu:
12474 : rq_unlock_irqrestore(rq, &rf);
12475 : }
12476 :
12477 : /* Idle groups have minimum weight. */
12478 : if (tg_is_idle(tg))
12479 : __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO));
12480 : else
12481 : __sched_group_set_shares(tg, NICE_0_LOAD);
12482 :
12483 : mutex_unlock(&shares_mutex);
12484 : return 0;
12485 : }
12486 :
12487 : #else /* CONFIG_FAIR_GROUP_SCHED */
12488 :
12489 0 : void free_fair_sched_group(struct task_group *tg) { }
12490 :
12491 0 : int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
12492 : {
12493 0 : return 1;
12494 : }
12495 :
12496 0 : void online_fair_sched_group(struct task_group *tg) { }
12497 :
12498 0 : void unregister_fair_sched_group(struct task_group *tg) { }
12499 :
12500 : #endif /* CONFIG_FAIR_GROUP_SCHED */
12501 :
12502 :
12503 0 : static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
12504 : {
12505 0 : struct sched_entity *se = &task->se;
12506 0 : unsigned int rr_interval = 0;
12507 :
12508 : /*
12509 : * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
12510 : * idle runqueue:
12511 : */
12512 0 : if (rq->cfs.load.weight)
12513 0 : rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se));
12514 :
12515 0 : return rr_interval;
12516 : }
12517 :
12518 : /*
12519 : * All the scheduling class methods:
12520 : */
12521 : DEFINE_SCHED_CLASS(fair) = {
12522 :
12523 : .enqueue_task = enqueue_task_fair,
12524 : .dequeue_task = dequeue_task_fair,
12525 : .yield_task = yield_task_fair,
12526 : .yield_to_task = yield_to_task_fair,
12527 :
12528 : .check_preempt_curr = check_preempt_wakeup,
12529 :
12530 : .pick_next_task = __pick_next_task_fair,
12531 : .put_prev_task = put_prev_task_fair,
12532 : .set_next_task = set_next_task_fair,
12533 :
12534 : #ifdef CONFIG_SMP
12535 : .balance = balance_fair,
12536 : .pick_task = pick_task_fair,
12537 : .select_task_rq = select_task_rq_fair,
12538 : .migrate_task_rq = migrate_task_rq_fair,
12539 :
12540 : .rq_online = rq_online_fair,
12541 : .rq_offline = rq_offline_fair,
12542 :
12543 : .task_dead = task_dead_fair,
12544 : .set_cpus_allowed = set_cpus_allowed_common,
12545 : #endif
12546 :
12547 : .task_tick = task_tick_fair,
12548 : .task_fork = task_fork_fair,
12549 :
12550 : .prio_changed = prio_changed_fair,
12551 : .switched_from = switched_from_fair,
12552 : .switched_to = switched_to_fair,
12553 :
12554 : .get_rr_interval = get_rr_interval_fair,
12555 :
12556 : .update_curr = update_curr_fair,
12557 :
12558 : #ifdef CONFIG_FAIR_GROUP_SCHED
12559 : .task_change_group = task_change_group_fair,
12560 : #endif
12561 :
12562 : #ifdef CONFIG_UCLAMP_TASK
12563 : .uclamp_enabled = 1,
12564 : #endif
12565 : };
12566 :
12567 : #ifdef CONFIG_SCHED_DEBUG
12568 0 : void print_cfs_stats(struct seq_file *m, int cpu)
12569 : {
12570 : struct cfs_rq *cfs_rq, *pos;
12571 :
12572 : rcu_read_lock();
12573 0 : for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos)
12574 0 : print_cfs_rq(m, cpu, cfs_rq);
12575 : rcu_read_unlock();
12576 0 : }
12577 :
12578 : #ifdef CONFIG_NUMA_BALANCING
12579 : void show_numa_stats(struct task_struct *p, struct seq_file *m)
12580 : {
12581 : int node;
12582 : unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0;
12583 : struct numa_group *ng;
12584 :
12585 : rcu_read_lock();
12586 : ng = rcu_dereference(p->numa_group);
12587 : for_each_online_node(node) {
12588 : if (p->numa_faults) {
12589 : tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)];
12590 : tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)];
12591 : }
12592 : if (ng) {
12593 : gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)],
12594 : gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
12595 : }
12596 : print_numa_stats(m, node, tsf, tpf, gsf, gpf);
12597 : }
12598 : rcu_read_unlock();
12599 : }
12600 : #endif /* CONFIG_NUMA_BALANCING */
12601 : #endif /* CONFIG_SCHED_DEBUG */
12602 :
12603 1 : __init void init_sched_fair_class(void)
12604 : {
12605 : #ifdef CONFIG_SMP
12606 : int i;
12607 :
12608 : for_each_possible_cpu(i) {
12609 : zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i));
12610 : zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i));
12611 :
12612 : #ifdef CONFIG_CFS_BANDWIDTH
12613 : INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i));
12614 : INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list);
12615 : #endif
12616 : }
12617 :
12618 : open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
12619 :
12620 : #ifdef CONFIG_NO_HZ_COMMON
12621 : nohz.next_balance = jiffies;
12622 : nohz.next_blocked = jiffies;
12623 : zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
12624 : #endif
12625 : #endif /* SMP */
12626 :
12627 1 : }
|
