Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _LINUX_PART_STAT_H
       3             : #define _LINUX_PART_STAT_H
       4             : 
       5             : #include <linux/blkdev.h>
       6             : #include <asm/local.h>
       7             : 
       8             : struct disk_stats {
       9             :         u64 nsecs[NR_STAT_GROUPS];
      10             :         unsigned long sectors[NR_STAT_GROUPS];
      11             :         unsigned long ios[NR_STAT_GROUPS];
      12             :         unsigned long merges[NR_STAT_GROUPS];
      13             :         unsigned long io_ticks;
      14             :         local_t in_flight[2];
      15             : };
      16             : 
      17             : /*
      18             :  * Macros to operate on percpu disk statistics:
      19             :  *
      20             :  * {disk|part|all}_stat_{add|sub|inc|dec}() modify the stat counters and should
      21             :  * be called between disk_stat_lock() and disk_stat_unlock().
      22             :  *
      23             :  * part_stat_read() can be called at any time.
      24             :  */
      25             : #define part_stat_lock()        preempt_disable()
      26             : #define part_stat_unlock()      preempt_enable()
      27             : 
      28             : #define part_stat_get_cpu(part, field, cpu)                             \
      29             :         (per_cpu_ptr((part)->bd_stats, (cpu))->field)
      30             : 
      31             : #define part_stat_get(part, field)                                      \
      32             :         part_stat_get_cpu(part, field, smp_processor_id())
      33             : 
      34             : #define part_stat_read(part, field)                                     \
      35             : ({                                                                      \
      36             :         typeof((part)->bd_stats->field) res = 0;                  \
      37             :         unsigned int _cpu;                                              \
      38             :         for_each_possible_cpu(_cpu)                                     \
      39             :                 res += per_cpu_ptr((part)->bd_stats, _cpu)->field; \
      40             :         res;                                                            \
      41             : })
      42             : 
      43             : static inline void part_stat_set_all(struct block_device *part, int value)
      44             : {
      45             :         int i;
      46             : 
      47           0 :         for_each_possible_cpu(i)
      48           0 :                 memset(per_cpu_ptr(part->bd_stats, i), value,
      49             :                                 sizeof(struct disk_stats));
      50             : }
      51             : 
      52             : #define part_stat_read_accum(part, field)                               \
      53             :         (part_stat_read(part, field[STAT_READ]) +                       \
      54             :          part_stat_read(part, field[STAT_WRITE]) +                      \
      55             :          part_stat_read(part, field[STAT_DISCARD]))
      56             : 
      57             : #define __part_stat_add(part, field, addnd)                             \
      58             :         __this_cpu_add((part)->bd_stats->field, addnd)
      59             : 
      60             : #define part_stat_add(part, field, addnd)       do {                    \
      61             :         __part_stat_add((part), field, addnd);                          \
      62             :         if ((part)->bd_partno)                                               \
      63             :                 __part_stat_add(bdev_whole(part), field, addnd);        \
      64             : } while (0)
      65             : 
      66             : #define part_stat_dec(part, field)                                      \
      67             :         part_stat_add(part, field, -1)
      68             : #define part_stat_inc(part, field)                                      \
      69             :         part_stat_add(part, field, 1)
      70             : #define part_stat_sub(part, field, subnd)                               \
      71             :         part_stat_add(part, field, -subnd)
      72             : 
      73             : #define part_stat_local_dec(part, field)                                \
      74             :         local_dec(&(part_stat_get(part, field)))
      75             : #define part_stat_local_inc(part, field)                                \
      76             :         local_inc(&(part_stat_get(part, field)))
      77             : #define part_stat_local_read(part, field)                               \
      78             :         local_read(&(part_stat_get(part, field)))
      79             : #define part_stat_local_read_cpu(part, field, cpu)                      \
      80             :         local_read(&(part_stat_get_cpu(part, field, cpu)))
      81             : 
      82             : #endif /* _LINUX_PART_STAT_H */