Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _LINUX_SCHED_IDLE_H
       3             : #define _LINUX_SCHED_IDLE_H
       4             : 
       5             : #include <linux/sched.h>
       6             : 
       7             : enum cpu_idle_type {
       8             :         CPU_IDLE,
       9             :         CPU_NOT_IDLE,
      10             :         CPU_NEWLY_IDLE,
      11             :         CPU_MAX_IDLE_TYPES
      12             : };
      13             : 
      14             : #ifdef CONFIG_SMP
      15             : extern void wake_up_if_idle(int cpu);
      16             : #else
      17             : static inline void wake_up_if_idle(int cpu) { }
      18             : #endif
      19             : 
      20             : /*
      21             :  * Idle thread specific functions to determine the need_resched
      22             :  * polling state.
      23             :  */
      24             : #ifdef TIF_POLLING_NRFLAG
      25             : 
      26             : #ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H
      27             : 
      28             : static __always_inline void __current_set_polling(void)
      29             : {
      30             :         arch_set_bit(TIF_POLLING_NRFLAG,
      31             :                      (unsigned long *)(&current_thread_info()->flags));
      32             : }
      33             : 
      34             : static __always_inline void __current_clr_polling(void)
      35             : {
      36             :         arch_clear_bit(TIF_POLLING_NRFLAG,
      37             :                        (unsigned long *)(&current_thread_info()->flags));
      38             : }
      39             : 
      40             : #else
      41             : 
      42             : static __always_inline void __current_set_polling(void)
      43             : {
      44             :         set_bit(TIF_POLLING_NRFLAG,
      45             :                 (unsigned long *)(&current_thread_info()->flags));
      46             : }
      47             : 
      48             : static __always_inline void __current_clr_polling(void)
      49             : {
      50             :         clear_bit(TIF_POLLING_NRFLAG,
      51             :                   (unsigned long *)(&current_thread_info()->flags));
      52             : }
      53             : 
      54             : #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H */
      55             : 
      56             : static __always_inline bool __must_check current_set_polling_and_test(void)
      57             : {
      58             :         __current_set_polling();
      59             : 
      60             :         /*
      61             :          * Polling state must be visible before we test NEED_RESCHED,
      62             :          * paired by resched_curr()
      63             :          */
      64             :         smp_mb__after_atomic();
      65             : 
      66             :         return unlikely(tif_need_resched());
      67             : }
      68             : 
      69             : static __always_inline bool __must_check current_clr_polling_and_test(void)
      70             : {
      71             :         __current_clr_polling();
      72             : 
      73             :         /*
      74             :          * Polling state must be visible before we test NEED_RESCHED,
      75             :          * paired by resched_curr()
      76             :          */
      77             :         smp_mb__after_atomic();
      78             : 
      79             :         return unlikely(tif_need_resched());
      80             : }
      81             : 
      82             : #else
      83             : static inline void __current_set_polling(void) { }
      84             : static inline void __current_clr_polling(void) { }
      85             : 
      86             : static inline bool __must_check current_set_polling_and_test(void)
      87             : {
      88             :         return unlikely(tif_need_resched());
      89             : }
      90             : static inline bool __must_check current_clr_polling_and_test(void)
      91             : {
      92           0 :         return unlikely(tif_need_resched());
      93             : }
      94             : #endif
      95             : 
      96             : static __always_inline void current_clr_polling(void)
      97             : {
      98             :         __current_clr_polling();
      99             : 
     100             :         /*
     101             :          * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
     102             :          * Once the bit is cleared, we'll get IPIs with every new
     103             :          * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
     104             :          * fold.
     105             :          */
     106             :         smp_mb(); /* paired with resched_curr() */
     107             : 
     108             :         preempt_fold_need_resched();
     109             : }
     110             : 
     111             : #endif /* _LINUX_SCHED_IDLE_H */