Line data    Source code

       1             : #ifdef CONFIG_SMP
       2             : #include "sched-pelt.h"
       3             : 
       4             : int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
       5             : int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
       6             : int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
       7             : int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
       8             : int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
       9             : 
      10             : #ifdef CONFIG_SCHED_THERMAL_PRESSURE
      11             : int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
      12             : 
      13             : static inline u64 thermal_load_avg(struct rq *rq)
      14             : {
      15             :         return READ_ONCE(rq->avg_thermal.load_avg);
      16             : }
      17             : #else
      18             : static inline int
      19             : update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
      20             : {
      21             :         return 0;
      22             : }
      23             : 
      24             : static inline u64 thermal_load_avg(struct rq *rq)
      25             : {
      26             :         return 0;
      27             : }
      28             : #endif
      29             : 
      30             : #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
      31             : int update_irq_load_avg(struct rq *rq, u64 running);
      32             : #else
      33             : static inline int
      34             : update_irq_load_avg(struct rq *rq, u64 running)
      35             : {
      36             :         return 0;
      37             : }
      38             : #endif
      39             : 
      40             : #define PELT_MIN_DIVIDER        (LOAD_AVG_MAX - 1024)
      41             : 
      42             : static inline u32 get_pelt_divider(struct sched_avg *avg)
      43             : {
      44             :         return PELT_MIN_DIVIDER + avg->period_contrib;
      45             : }
      46             : 
      47             : static inline void cfs_se_util_change(struct sched_avg *avg)
      48             : {
      49             :         unsigned int enqueued;
      50             : 
      51             :         if (!sched_feat(UTIL_EST))
      52             :                 return;
      53             : 
      54             :         /* Avoid store if the flag has been already reset */
      55             :         enqueued = avg->util_est.enqueued;
      56             :         if (!(enqueued & UTIL_AVG_UNCHANGED))
      57             :                 return;
      58             : 
      59             :         /* Reset flag to report util_avg has been updated */
      60             :         enqueued &= ~UTIL_AVG_UNCHANGED;
      61             :         WRITE_ONCE(avg->util_est.enqueued, enqueued);
      62             : }
      63             : 
      64             : static inline u64 rq_clock_pelt(struct rq *rq)
      65             : {
      66             :         lockdep_assert_rq_held(rq);
      67             :         assert_clock_updated(rq);
      68             : 
      69             :         return rq->clock_pelt - rq->lost_idle_time;
      70             : }
      71             : 
      72             : /* The rq is idle, we can sync to clock_task */
      73             : static inline void _update_idle_rq_clock_pelt(struct rq *rq)
      74             : {
      75             :         rq->clock_pelt  = rq_clock_task(rq);
      76             : 
      77             :         u64_u32_store(rq->clock_idle, rq_clock(rq));
      78             :         /* Paired with smp_rmb in migrate_se_pelt_lag() */
      79             :         smp_wmb();
      80             :         u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
      81             : }
      82             : 
      83             : /*
      84             :  * The clock_pelt scales the time to reflect the effective amount of
      85             :  * computation done during the running delta time but then sync back to
      86             :  * clock_task when rq is idle.
      87             :  *
      88             :  *
      89             :  * absolute time   | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
      90             :  * @ max capacity  ------******---------------******---------------
      91             :  * @ half capacity ------************---------************---------
      92             :  * clock pelt      | 1| 2|    3|    4| 7| 8| 9|   10|   11|14|15|16
      93             :  *
      94             :  */
      95             : static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
      96             : {
      97             :         if (unlikely(is_idle_task(rq->curr))) {
      98             :                 _update_idle_rq_clock_pelt(rq);
      99             :                 return;
     100             :         }
     101             : 
     102             :         /*
     103             :          * When a rq runs at a lower compute capacity, it will need
     104             :          * more time to do the same amount of work than at max
     105             :          * capacity. In order to be invariant, we scale the delta to
     106             :          * reflect how much work has been really done.
     107             :          * Running longer results in stealing idle time that will
     108             :          * disturb the load signal compared to max capacity. This
     109             :          * stolen idle time will be automatically reflected when the
     110             :          * rq will be idle and the clock will be synced with
     111             :          * rq_clock_task.
     112             :          */
     113             : 
     114             :         /*
     115             :          * Scale the elapsed time to reflect the real amount of
     116             :          * computation
     117             :          */
     118             :         delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
     119             :         delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
     120             : 
     121             :         rq->clock_pelt += delta;
     122             : }
     123             : 
     124             : /*
     125             :  * When rq becomes idle, we have to check if it has lost idle time
     126             :  * because it was fully busy. A rq is fully used when the /Sum util_sum
     127             :  * is greater or equal to:
     128             :  * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
     129             :  * For optimization and computing rounding purpose, we don't take into account
     130             :  * the position in the current window (period_contrib) and we use the higher
     131             :  * bound of util_sum to decide.
     132             :  */
     133             : static inline void update_idle_rq_clock_pelt(struct rq *rq)
     134             : {
     135             :         u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
     136             :         u32 util_sum = rq->cfs.avg.util_sum;
     137             :         util_sum += rq->avg_rt.util_sum;
     138             :         util_sum += rq->avg_dl.util_sum;
     139             : 
     140             :         /*
     141             :          * Reflecting stolen time makes sense only if the idle
     142             :          * phase would be present at max capacity. As soon as the
     143             :          * utilization of a rq has reached the maximum value, it is
     144             :          * considered as an always running rq without idle time to
     145             :          * steal. This potential idle time is considered as lost in
     146             :          * this case. We keep track of this lost idle time compare to
     147             :          * rq's clock_task.
     148             :          */
     149             :         if (util_sum >= divider)
     150             :                 rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
     151             : 
     152             :         _update_idle_rq_clock_pelt(rq);
     153             : }
     154             : 
     155             : #ifdef CONFIG_CFS_BANDWIDTH
     156             : static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
     157             : {
     158             :         u64 throttled;
     159             : 
     160             :         if (unlikely(cfs_rq->throttle_count))
     161             :                 throttled = U64_MAX;
     162             :         else
     163             :                 throttled = cfs_rq->throttled_clock_pelt_time;
     164             : 
     165             :         u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
     166             : }
     167             : 
     168             : /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
     169             : static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
     170             : {
     171             :         if (unlikely(cfs_rq->throttle_count))
     172             :                 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
     173             : 
     174             :         return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
     175             : }
     176             : #else
     177             : static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
     178             : static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
     179             : {
     180             :         return rq_clock_pelt(rq_of(cfs_rq));
     181             : }
     182             : #endif
     183             : 
     184             : #else
     185             : 
     186             : static inline int
     187             : update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
     188             : {
     189             :         return 0;
     190             : }
     191             : 
     192             : static inline int
     193             : update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
     194             : {
     195             :         return 0;
     196             : }
     197             : 
     198             : static inline int
     199             : update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
     200             : {
     201             :         return 0;
     202             : }
     203             : 
     204             : static inline int
     205             : update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
     206             : {
     207             :         return 0;
     208             : }
     209             : 
     210             : static inline u64 thermal_load_avg(struct rq *rq)
     211             : {
     212             :         return 0;
     213             : }
     214             : 
     215             : static inline int
     216             : update_irq_load_avg(struct rq *rq, u64 running)
     217             : {
     218             :         return 0;
     219             : }
     220             : 
     221             : static inline u64 rq_clock_pelt(struct rq *rq)
     222             : {
     223           0 :         return rq_clock_task(rq);
     224             : }
     225             : 
     226             : static inline void
     227             : update_rq_clock_pelt(struct rq *rq, s64 delta) { }
     228             : 
     229             : static inline void
     230             : update_idle_rq_clock_pelt(struct rq *rq) { }
     231             : 
     232             : static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
     233             : #endif
     234             : 
     235             :