Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Block stat tracking code
       4             :  *
       5             :  * Copyright (C) 2016 Jens Axboe
       6             :  */
       7             : #include <linux/kernel.h>
       8             : #include <linux/rculist.h>
       9             : 
      10             : #include "blk-stat.h"
      11             : #include "blk-mq.h"
      12             : #include "blk.h"
      13             : 
      14             : struct blk_queue_stats {
      15             :         struct list_head callbacks;
      16             :         spinlock_t lock;
      17             :         int accounting;
      18             : };
      19             : 
      20           0 : void blk_rq_stat_init(struct blk_rq_stat *stat)
      21             : {
      22           0 :         stat->min = -1ULL;
      23           0 :         stat->max = stat->nr_samples = stat->mean = 0;
      24           0 :         stat->batch = 0;
      25           0 : }
      26             : 
      27             : /* src is a per-cpu stat, mean isn't initialized */
      28           0 : void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
      29             : {
      30           0 :         if (!src->nr_samples)
      31             :                 return;
      32             : 
      33           0 :         dst->min = min(dst->min, src->min);
      34           0 :         dst->max = max(dst->max, src->max);
      35             : 
      36           0 :         dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples,
      37           0 :                                 dst->nr_samples + src->nr_samples);
      38             : 
      39           0 :         dst->nr_samples += src->nr_samples;
      40             : }
      41             : 
      42           0 : void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value)
      43             : {
      44           0 :         stat->min = min(stat->min, value);
      45           0 :         stat->max = max(stat->max, value);
      46           0 :         stat->batch += value;
      47           0 :         stat->nr_samples++;
      48           0 : }
      49             : 
      50           0 : void blk_stat_add(struct request *rq, u64 now)
      51             : {
      52           0 :         struct request_queue *q = rq->q;
      53             :         struct blk_stat_callback *cb;
      54             :         struct blk_rq_stat *stat;
      55             :         int bucket, cpu;
      56             :         u64 value;
      57             : 
      58           0 :         value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
      59             : 
      60           0 :         if (req_op(rq) == REQ_OP_READ || req_op(rq) == REQ_OP_WRITE)
      61             :                 blk_throtl_stat_add(rq, value);
      62             : 
      63             :         rcu_read_lock();
      64           0 :         cpu = get_cpu();
      65           0 :         list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
      66           0 :                 if (!blk_stat_is_active(cb))
      67           0 :                         continue;
      68             : 
      69           0 :                 bucket = cb->bucket_fn(rq);
      70           0 :                 if (bucket < 0)
      71           0 :                         continue;
      72             : 
      73           0 :                 stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];
      74             :                 blk_rq_stat_add(stat, value);
      75             :         }
      76           0 :         put_cpu();
      77             :         rcu_read_unlock();
      78           0 : }
      79             : 
      80           0 : static void blk_stat_timer_fn(struct timer_list *t)
      81             : {
      82           0 :         struct blk_stat_callback *cb = from_timer(cb, t, timer);
      83             :         unsigned int bucket;
      84             :         int cpu;
      85             : 
      86           0 :         for (bucket = 0; bucket < cb->buckets; bucket++)
      87           0 :                 blk_rq_stat_init(&cb->stat[bucket]);
      88             : 
      89           0 :         for_each_online_cpu(cpu) {
      90             :                 struct blk_rq_stat *cpu_stat;
      91             : 
      92           0 :                 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
      93           0 :                 for (bucket = 0; bucket < cb->buckets; bucket++) {
      94           0 :                         blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
      95           0 :                         blk_rq_stat_init(&cpu_stat[bucket]);
      96             :                 }
      97             :         }
      98             : 
      99           0 :         cb->timer_fn(cb);
     100           0 : }
     101             : 
     102             : struct blk_stat_callback *
     103           0 : blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
     104             :                         int (*bucket_fn)(const struct request *),
     105             :                         unsigned int buckets, void *data)
     106             : {
     107             :         struct blk_stat_callback *cb;
     108             : 
     109           0 :         cb = kmalloc(sizeof(*cb), GFP_KERNEL);
     110           0 :         if (!cb)
     111             :                 return NULL;
     112             : 
     113           0 :         cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
     114             :                                  GFP_KERNEL);
     115           0 :         if (!cb->stat) {
     116           0 :                 kfree(cb);
     117           0 :                 return NULL;
     118             :         }
     119           0 :         cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
     120             :                                       __alignof__(struct blk_rq_stat));
     121           0 :         if (!cb->cpu_stat) {
     122           0 :                 kfree(cb->stat);
     123           0 :                 kfree(cb);
     124           0 :                 return NULL;
     125             :         }
     126             : 
     127           0 :         cb->timer_fn = timer_fn;
     128           0 :         cb->bucket_fn = bucket_fn;
     129           0 :         cb->data = data;
     130           0 :         cb->buckets = buckets;
     131           0 :         timer_setup(&cb->timer, blk_stat_timer_fn, 0);
     132             : 
     133           0 :         return cb;
     134             : }
     135             : 
     136           0 : void blk_stat_add_callback(struct request_queue *q,
     137             :                            struct blk_stat_callback *cb)
     138             : {
     139             :         unsigned int bucket;
     140             :         unsigned long flags;
     141             :         int cpu;
     142             : 
     143           0 :         for_each_possible_cpu(cpu) {
     144             :                 struct blk_rq_stat *cpu_stat;
     145             : 
     146           0 :                 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
     147           0 :                 for (bucket = 0; bucket < cb->buckets; bucket++)
     148           0 :                         blk_rq_stat_init(&cpu_stat[bucket]);
     149             :         }
     150             : 
     151           0 :         spin_lock_irqsave(&q->stats->lock, flags);
     152           0 :         list_add_tail_rcu(&cb->list, &q->stats->callbacks);
     153           0 :         blk_queue_flag_set(QUEUE_FLAG_STATS, q);
     154           0 :         spin_unlock_irqrestore(&q->stats->lock, flags);
     155           0 : }
     156             : 
     157           0 : void blk_stat_remove_callback(struct request_queue *q,
     158             :                               struct blk_stat_callback *cb)
     159             : {
     160             :         unsigned long flags;
     161             : 
     162           0 :         spin_lock_irqsave(&q->stats->lock, flags);
     163           0 :         list_del_rcu(&cb->list);
     164           0 :         if (list_empty(&q->stats->callbacks) && !q->stats->accounting)
     165           0 :                 blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
     166           0 :         spin_unlock_irqrestore(&q->stats->lock, flags);
     167             : 
     168           0 :         del_timer_sync(&cb->timer);
     169           0 : }
     170             : 
     171           0 : static void blk_stat_free_callback_rcu(struct rcu_head *head)
     172             : {
     173             :         struct blk_stat_callback *cb;
     174             : 
     175           0 :         cb = container_of(head, struct blk_stat_callback, rcu);
     176           0 :         free_percpu(cb->cpu_stat);
     177           0 :         kfree(cb->stat);
     178           0 :         kfree(cb);
     179           0 : }
     180             : 
     181           0 : void blk_stat_free_callback(struct blk_stat_callback *cb)
     182             : {
     183           0 :         if (cb)
     184           0 :                 call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
     185           0 : }
     186             : 
     187           0 : void blk_stat_disable_accounting(struct request_queue *q)
     188             : {
     189             :         unsigned long flags;
     190             : 
     191           0 :         spin_lock_irqsave(&q->stats->lock, flags);
     192           0 :         if (!--q->stats->accounting && list_empty(&q->stats->callbacks))
     193           0 :                 blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
     194           0 :         spin_unlock_irqrestore(&q->stats->lock, flags);
     195           0 : }
     196             : EXPORT_SYMBOL_GPL(blk_stat_disable_accounting);
     197             : 
     198           0 : void blk_stat_enable_accounting(struct request_queue *q)
     199             : {
     200             :         unsigned long flags;
     201             : 
     202           0 :         spin_lock_irqsave(&q->stats->lock, flags);
     203           0 :         if (!q->stats->accounting++ && list_empty(&q->stats->callbacks))
     204           0 :                 blk_queue_flag_set(QUEUE_FLAG_STATS, q);
     205           0 :         spin_unlock_irqrestore(&q->stats->lock, flags);
     206           0 : }
     207             : EXPORT_SYMBOL_GPL(blk_stat_enable_accounting);
     208             : 
     209           0 : struct blk_queue_stats *blk_alloc_queue_stats(void)
     210             : {
     211             :         struct blk_queue_stats *stats;
     212             : 
     213           0 :         stats = kmalloc(sizeof(*stats), GFP_KERNEL);
     214           0 :         if (!stats)
     215             :                 return NULL;
     216             : 
     217           0 :         INIT_LIST_HEAD(&stats->callbacks);
     218           0 :         spin_lock_init(&stats->lock);
     219           0 :         stats->accounting = 0;
     220             : 
     221           0 :         return stats;
     222             : }
     223             : 
     224           0 : void blk_free_queue_stats(struct blk_queue_stats *stats)
     225             : {
     226           0 :         if (!stats)
     227             :                 return;
     228             : 
     229           0 :         WARN_ON(!list_empty(&stats->callbacks));
     230             : 
     231           0 :         kfree(stats);
     232             : }