Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
       4             :  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
       5             :  *
       6             :  * This file contains the core interrupt handling code. Detailed
       7             :  * information is available in Documentation/core-api/genericirq.rst
       8             :  *
       9             :  */
      10             : 
      11             : #include <linux/irq.h>
      12             : #include <linux/random.h>
      13             : #include <linux/sched.h>
      14             : #include <linux/interrupt.h>
      15             : #include <linux/kernel_stat.h>
      16             : 
      17             : #include <asm/irq_regs.h>
      18             : 
      19             : #include <trace/events/irq.h>
      20             : 
      21             : #include "internals.h"
      22             : 
      23             : #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
      24             : void (*handle_arch_irq)(struct pt_regs *) __ro_after_init;
      25             : #endif
      26             : 
      27             : /**
      28             :  * handle_bad_irq - handle spurious and unhandled irqs
      29             :  * @desc:      description of the interrupt
      30             :  *
      31             :  * Handles spurious and unhandled IRQ's. It also prints a debugmessage.
      32             :  */
      33           0 : void handle_bad_irq(struct irq_desc *desc)
      34             : {
      35           0 :         unsigned int irq = irq_desc_get_irq(desc);
      36             : 
      37           0 :         print_irq_desc(irq, desc);
      38           0 :         kstat_incr_irqs_this_cpu(desc);
      39           0 :         ack_bad_irq(irq);
      40           0 : }
      41             : EXPORT_SYMBOL_GPL(handle_bad_irq);
      42             : 
      43             : /*
      44             :  * Special, empty irq handler:
      45             :  */
      46           0 : irqreturn_t no_action(int cpl, void *dev_id)
      47             : {
      48           0 :         return IRQ_NONE;
      49             : }
      50             : EXPORT_SYMBOL_GPL(no_action);
      51             : 
      52             : static void warn_no_thread(unsigned int irq, struct irqaction *action)
      53             : {
      54           0 :         if (test_and_set_bit(IRQTF_WARNED, &action->thread_flags))
      55             :                 return;
      56             : 
      57           0 :         printk(KERN_WARNING "IRQ %d device %s returned IRQ_WAKE_THREAD "
      58             :                "but no thread function available.", irq, action->name);
      59             : }
      60             : 
      61           0 : void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action)
      62             : {
      63             :         /*
      64             :          * In case the thread crashed and was killed we just pretend that
      65             :          * we handled the interrupt. The hardirq handler has disabled the
      66             :          * device interrupt, so no irq storm is lurking.
      67             :          */
      68           0 :         if (action->thread->flags & PF_EXITING)
      69             :                 return;
      70             : 
      71             :         /*
      72             :          * Wake up the handler thread for this action. If the
      73             :          * RUNTHREAD bit is already set, nothing to do.
      74             :          */
      75           0 :         if (test_and_set_bit(IRQTF_RUNTHREAD, &action->thread_flags))
      76             :                 return;
      77             : 
      78             :         /*
      79             :          * It's safe to OR the mask lockless here. We have only two
      80             :          * places which write to threads_oneshot: This code and the
      81             :          * irq thread.
      82             :          *
      83             :          * This code is the hard irq context and can never run on two
      84             :          * cpus in parallel. If it ever does we have more serious
      85             :          * problems than this bitmask.
      86             :          *
      87             :          * The irq threads of this irq which clear their "running" bit
      88             :          * in threads_oneshot are serialized via desc->lock against
      89             :          * each other and they are serialized against this code by
      90             :          * IRQS_INPROGRESS.
      91             :          *
      92             :          * Hard irq handler:
      93             :          *
      94             :          *      spin_lock(desc->lock);
      95             :          *      desc->state |= IRQS_INPROGRESS;
      96             :          *      spin_unlock(desc->lock);
      97             :          *      set_bit(IRQTF_RUNTHREAD, &action->thread_flags);
      98             :          *      desc->threads_oneshot |= mask;
      99             :          *      spin_lock(desc->lock);
     100             :          *      desc->state &= ~IRQS_INPROGRESS;
     101             :          *      spin_unlock(desc->lock);
     102             :          *
     103             :          * irq thread:
     104             :          *
     105             :          * again:
     106             :          *      spin_lock(desc->lock);
     107             :          *      if (desc->state & IRQS_INPROGRESS) {
     108             :          *              spin_unlock(desc->lock);
     109             :          *              while(desc->state & IRQS_INPROGRESS)
     110             :          *                      cpu_relax();
     111             :          *              goto again;
     112             :          *      }
     113             :          *      if (!test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
     114             :          *              desc->threads_oneshot &= ~mask;
     115             :          *      spin_unlock(desc->lock);
     116             :          *
     117             :          * So either the thread waits for us to clear IRQS_INPROGRESS
     118             :          * or we are waiting in the flow handler for desc->lock to be
     119             :          * released before we reach this point. The thread also checks
     120             :          * IRQTF_RUNTHREAD under desc->lock. If set it leaves
     121             :          * threads_oneshot untouched and runs the thread another time.
     122             :          */
     123           0 :         desc->threads_oneshot |= action->thread_mask;
     124             : 
     125             :         /*
     126             :          * We increment the threads_active counter in case we wake up
     127             :          * the irq thread. The irq thread decrements the counter when
     128             :          * it returns from the handler or in the exit path and wakes
     129             :          * up waiters which are stuck in synchronize_irq() when the
     130             :          * active count becomes zero. synchronize_irq() is serialized
     131             :          * against this code (hard irq handler) via IRQS_INPROGRESS
     132             :          * like the finalize_oneshot() code. See comment above.
     133             :          */
     134           0 :         atomic_inc(&desc->threads_active);
     135             : 
     136           0 :         wake_up_process(action->thread);
     137             : }
     138             : 
     139        2723 : irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc)
     140             : {
     141        2723 :         irqreturn_t retval = IRQ_NONE;
     142        2723 :         unsigned int irq = desc->irq_data.irq;
     143             :         struct irqaction *action;
     144             : 
     145        2723 :         record_irq_time(desc);
     146             : 
     147        5446 :         for_each_action_of_desc(desc, action) {
     148             :                 irqreturn_t res;
     149             : 
     150             :                 /*
     151             :                  * If this IRQ would be threaded under force_irqthreads, mark it so.
     152             :                  */
     153        2723 :                 if (irq_settings_can_thread(desc) &&
     154             :                     !(action->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT)))
     155             :                         lockdep_hardirq_threaded();
     156             : 
     157        2723 :                 trace_irq_handler_entry(irq, action);
     158        2723 :                 res = action->handler(irq, action->dev_id);
     159        2723 :                 trace_irq_handler_exit(irq, action, res);
     160             : 
     161        2723 :                 if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pS enabled interrupts\n",
     162             :                               irq, action->handler))
     163             :                         local_irq_disable();
     164             : 
     165        2723 :                 switch (res) {
     166             :                 case IRQ_WAKE_THREAD:
     167             :                         /*
     168             :                          * Catch drivers which return WAKE_THREAD but
     169             :                          * did not set up a thread function
     170             :                          */
     171           0 :                         if (unlikely(!action->thread_fn)) {
     172             :                                 warn_no_thread(irq, action);
     173             :                                 break;
     174             :                         }
     175             : 
     176           0 :                         __irq_wake_thread(desc, action);
     177           0 :                         break;
     178             : 
     179             :                 default:
     180             :                         break;
     181             :                 }
     182             : 
     183        2723 :                 retval |= res;
     184             :         }
     185             : 
     186        2723 :         return retval;
     187             : }
     188             : 
     189        2723 : irqreturn_t handle_irq_event_percpu(struct irq_desc *desc)
     190             : {
     191             :         irqreturn_t retval;
     192             : 
     193        2723 :         retval = __handle_irq_event_percpu(desc);
     194             : 
     195        2723 :         add_interrupt_randomness(desc->irq_data.irq);
     196             : 
     197        5446 :         if (!irq_settings_no_debug(desc))
     198        2723 :                 note_interrupt(desc, retval);
     199        2723 :         return retval;
     200             : }
     201             : 
     202        2723 : irqreturn_t handle_irq_event(struct irq_desc *desc)
     203             : {
     204             :         irqreturn_t ret;
     205             : 
     206        2723 :         desc->istate &= ~IRQS_PENDING;
     207        5446 :         irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
     208        2723 :         raw_spin_unlock(&desc->lock);
     209             : 
     210        2723 :         ret = handle_irq_event_percpu(desc);
     211             : 
     212        2723 :         raw_spin_lock(&desc->lock);
     213        5446 :         irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
     214        2723 :         return ret;
     215             : }
     216             : 
     217             : #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
     218             : int __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
     219             : {
     220             :         if (handle_arch_irq)
     221             :                 return -EBUSY;
     222             : 
     223             :         handle_arch_irq = handle_irq;
     224             :         return 0;
     225             : }
     226             : 
     227             : /**
     228             :  * generic_handle_arch_irq - root irq handler for architectures which do no
     229             :  *                           entry accounting themselves
     230             :  * @regs:       Register file coming from the low-level handling code
     231             :  */
     232             : asmlinkage void noinstr generic_handle_arch_irq(struct pt_regs *regs)
     233             : {
     234             :         struct pt_regs *old_regs;
     235             : 
     236             :         irq_enter();
     237             :         old_regs = set_irq_regs(regs);
     238             :         handle_arch_irq(regs);
     239             :         set_irq_regs(old_regs);
     240             :         irq_exit();
     241             : }
     242             : #endif