![Linus Torvalds](/assets/img/avatar_default.png)
Pull perf events changes for v3.4 from Ingo Molnar: - New "hardware based branch profiling" feature both on the kernel and the tooling side, on CPUs that support it. (modern x86 Intel CPUs with the 'LBR' hardware feature currently.) This new feature is basically a sophisticated 'magnifying glass' for branch execution - something that is pretty difficult to extract from regular, function histogram centric profiles. The simplest mode is activated via 'perf record -b', and the result looks like this in perf report: $ perf record -b any_call,u -e cycles:u branchy $ perf report -b --sort=symbol 52.34% [.] main [.] f1 24.04% [.] f1 [.] f3 23.60% [.] f1 [.] f2 0.01% [k] _IO_new_file_xsputn [k] _IO_file_overflow 0.01% [k] _IO_vfprintf_internal [k] _IO_new_file_xsputn 0.01% [k] _IO_vfprintf_internal [k] strchrnul 0.01% [k] __printf [k] _IO_vfprintf_internal 0.01% [k] main [k] __printf This output shows from/to branch columns and shows the highest percentage (from,to) jump combinations - i.e. the most likely taken branches in the system. "branches" can also include function calls and any other synchronous and asynchronous transitions of the instruction pointer that are not 'next instruction' - such as system calls, traps, interrupts, etc. This feature comes with (hopefully intuitive) flat ascii and TUI support in perf report. - Various 'perf annotate' visual improvements for us assembly junkies. It will now recognize function calls in the TUI and by hitting enter you can follow the call (recursively) and back, amongst other improvements. - Multiple threads/processes recording support in perf record, perf stat, perf top - which is activated via a comma-list of PIDs: perf top -p 21483,21485 perf stat -p 21483,21485 -ddd perf record -p 21483,21485 - Support for per UID views, via the --uid paramter to perf top, perf report, etc. For example 'perf top --uid mingo' will only show the tasks that I am running, excluding other users, root, etc. - Jump label restructurings and improvements - this includes the factoring out of the (hopefully much clearer) include/linux/static_key.h generic facility: struct static_key key = STATIC_KEY_INIT_FALSE; ... if (static_key_false(&key)) do unlikely code else do likely code ... static_key_slow_inc(); ... static_key_slow_inc(); ... The static_key_false() branch will be generated into the code with as little impact to the likely code path as possible. the static_key_slow_*() APIs flip the branch via live kernel code patching. This facility can now be used more widely within the kernel to micro-optimize hot branches whose likelihood matches the static-key usage and fast/slow cost patterns. - SW function tracer improvements: perf support and filtering support. - Various hardenings of the perf.data ABI, to make older perf.data's smoother on newer tool versions, to make new features integrate more smoothly, to support cross-endian recording/analyzing workflows better, etc. - Restructuring of the kprobes code, the splitting out of 'optprobes', and a corner case bugfix. - Allow the tracing of kernel console output (printk). - Improvements/fixes to user-space RDPMC support, allowing user-space self-profiling code to extract PMU counts without performing any system calls, while playing nice with the kernel side. - 'perf bench' improvements - ... and lots of internal restructurings, cleanups and fixes that made these features possible. And, as usual this list is incomplete as there were also lots of other improvements * 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (120 commits) perf report: Fix annotate double quit issue in branch view mode perf report: Remove duplicate annotate choice in branch view mode perf/x86: Prettify pmu config literals perf report: Enable TUI in branch view mode perf report: Auto-detect branch stack sampling mode perf record: Add HEADER_BRANCH_STACK tag perf record: Provide default branch stack sampling mode option perf tools: Make perf able to read files from older ABIs perf tools: Fix ABI compatibility bug in print_event_desc() perf tools: Enable reading of perf.data files from different ABI rev perf: Add ABI reference sizes perf report: Add support for taken branch sampling perf record: Add support for sampling taken branch perf tools: Add code to support PERF_SAMPLE_BRANCH_STACK x86/kprobes: Split out optprobe related code to kprobes-opt.c x86/kprobes: Fix a bug which can modify kernel code permanently x86/kprobes: Fix instruction recovery on optimized path perf: Add callback to flush branch_stack on context switch perf: Disable PERF_SAMPLE_BRANCH_* when not supported perf/x86: Add LBR software filter support for Intel CPUs ...
753 lines
19 KiB
C
753 lines
19 KiB
C
/*
|
|
* linux/kernel/irq/chip.c
|
|
*
|
|
* Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
|
|
* Copyright (C) 2005-2006, Thomas Gleixner, Russell King
|
|
*
|
|
* This file contains the core interrupt handling code, for irq-chip
|
|
* based architectures.
|
|
*
|
|
* Detailed information is available in Documentation/DocBook/genericirq
|
|
*/
|
|
|
|
#include <linux/irq.h>
|
|
#include <linux/msi.h>
|
|
#include <linux/module.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/kernel_stat.h>
|
|
|
|
#include <trace/events/irq.h>
|
|
|
|
#include "internals.h"
|
|
|
|
/**
|
|
* irq_set_chip - set the irq chip for an irq
|
|
* @irq: irq number
|
|
* @chip: pointer to irq chip description structure
|
|
*/
|
|
int irq_set_chip(unsigned int irq, struct irq_chip *chip)
|
|
{
|
|
unsigned long flags;
|
|
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
|
|
|
|
if (!desc)
|
|
return -EINVAL;
|
|
|
|
if (!chip)
|
|
chip = &no_irq_chip;
|
|
|
|
desc->irq_data.chip = chip;
|
|
irq_put_desc_unlock(desc, flags);
|
|
/*
|
|
* For !CONFIG_SPARSE_IRQ make the irq show up in
|
|
* allocated_irqs. For the CONFIG_SPARSE_IRQ case, it is
|
|
* already marked, and this call is harmless.
|
|
*/
|
|
irq_reserve_irq(irq);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(irq_set_chip);
|
|
|
|
/**
|
|
* irq_set_type - set the irq trigger type for an irq
|
|
* @irq: irq number
|
|
* @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
|
|
*/
|
|
int irq_set_irq_type(unsigned int irq, unsigned int type)
|
|
{
|
|
unsigned long flags;
|
|
struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
|
|
int ret = 0;
|
|
|
|
if (!desc)
|
|
return -EINVAL;
|
|
|
|
type &= IRQ_TYPE_SENSE_MASK;
|
|
ret = __irq_set_trigger(desc, irq, type);
|
|
irq_put_desc_busunlock(desc, flags);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(irq_set_irq_type);
|
|
|
|
/**
|
|
* irq_set_handler_data - set irq handler data for an irq
|
|
* @irq: Interrupt number
|
|
* @data: Pointer to interrupt specific data
|
|
*
|
|
* Set the hardware irq controller data for an irq
|
|
*/
|
|
int irq_set_handler_data(unsigned int irq, void *data)
|
|
{
|
|
unsigned long flags;
|
|
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
|
|
|
|
if (!desc)
|
|
return -EINVAL;
|
|
desc->irq_data.handler_data = data;
|
|
irq_put_desc_unlock(desc, flags);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(irq_set_handler_data);
|
|
|
|
/**
|
|
* irq_set_msi_desc - set MSI descriptor data for an irq
|
|
* @irq: Interrupt number
|
|
* @entry: Pointer to MSI descriptor data
|
|
*
|
|
* Set the MSI descriptor entry for an irq
|
|
*/
|
|
int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
|
|
{
|
|
unsigned long flags;
|
|
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
|
|
|
|
if (!desc)
|
|
return -EINVAL;
|
|
desc->irq_data.msi_desc = entry;
|
|
if (entry)
|
|
entry->irq = irq;
|
|
irq_put_desc_unlock(desc, flags);
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* irq_set_chip_data - set irq chip data for an irq
|
|
* @irq: Interrupt number
|
|
* @data: Pointer to chip specific data
|
|
*
|
|
* Set the hardware irq chip data for an irq
|
|
*/
|
|
int irq_set_chip_data(unsigned int irq, void *data)
|
|
{
|
|
unsigned long flags;
|
|
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
|
|
|
|
if (!desc)
|
|
return -EINVAL;
|
|
desc->irq_data.chip_data = data;
|
|
irq_put_desc_unlock(desc, flags);
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(irq_set_chip_data);
|
|
|
|
struct irq_data *irq_get_irq_data(unsigned int irq)
|
|
{
|
|
struct irq_desc *desc = irq_to_desc(irq);
|
|
|
|
return desc ? &desc->irq_data : NULL;
|
|
}
|
|
EXPORT_SYMBOL_GPL(irq_get_irq_data);
|
|
|
|
static void irq_state_clr_disabled(struct irq_desc *desc)
|
|
{
|
|
irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
|
|
}
|
|
|
|
static void irq_state_set_disabled(struct irq_desc *desc)
|
|
{
|
|
irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
|
|
}
|
|
|
|
static void irq_state_clr_masked(struct irq_desc *desc)
|
|
{
|
|
irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
|
|
}
|
|
|
|
static void irq_state_set_masked(struct irq_desc *desc)
|
|
{
|
|
irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
|
|
}
|
|
|
|
int irq_startup(struct irq_desc *desc, bool resend)
|
|
{
|
|
int ret = 0;
|
|
|
|
irq_state_clr_disabled(desc);
|
|
desc->depth = 0;
|
|
|
|
if (desc->irq_data.chip->irq_startup) {
|
|
ret = desc->irq_data.chip->irq_startup(&desc->irq_data);
|
|
irq_state_clr_masked(desc);
|
|
} else {
|
|
irq_enable(desc);
|
|
}
|
|
if (resend)
|
|
check_irq_resend(desc, desc->irq_data.irq);
|
|
return ret;
|
|
}
|
|
|
|
void irq_shutdown(struct irq_desc *desc)
|
|
{
|
|
irq_state_set_disabled(desc);
|
|
desc->depth = 1;
|
|
if (desc->irq_data.chip->irq_shutdown)
|
|
desc->irq_data.chip->irq_shutdown(&desc->irq_data);
|
|
else if (desc->irq_data.chip->irq_disable)
|
|
desc->irq_data.chip->irq_disable(&desc->irq_data);
|
|
else
|
|
desc->irq_data.chip->irq_mask(&desc->irq_data);
|
|
irq_state_set_masked(desc);
|
|
}
|
|
|
|
void irq_enable(struct irq_desc *desc)
|
|
{
|
|
irq_state_clr_disabled(desc);
|
|
if (desc->irq_data.chip->irq_enable)
|
|
desc->irq_data.chip->irq_enable(&desc->irq_data);
|
|
else
|
|
desc->irq_data.chip->irq_unmask(&desc->irq_data);
|
|
irq_state_clr_masked(desc);
|
|
}
|
|
|
|
void irq_disable(struct irq_desc *desc)
|
|
{
|
|
irq_state_set_disabled(desc);
|
|
if (desc->irq_data.chip->irq_disable) {
|
|
desc->irq_data.chip->irq_disable(&desc->irq_data);
|
|
irq_state_set_masked(desc);
|
|
}
|
|
}
|
|
|
|
void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
|
|
{
|
|
if (desc->irq_data.chip->irq_enable)
|
|
desc->irq_data.chip->irq_enable(&desc->irq_data);
|
|
else
|
|
desc->irq_data.chip->irq_unmask(&desc->irq_data);
|
|
cpumask_set_cpu(cpu, desc->percpu_enabled);
|
|
}
|
|
|
|
void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
|
|
{
|
|
if (desc->irq_data.chip->irq_disable)
|
|
desc->irq_data.chip->irq_disable(&desc->irq_data);
|
|
else
|
|
desc->irq_data.chip->irq_mask(&desc->irq_data);
|
|
cpumask_clear_cpu(cpu, desc->percpu_enabled);
|
|
}
|
|
|
|
static inline void mask_ack_irq(struct irq_desc *desc)
|
|
{
|
|
if (desc->irq_data.chip->irq_mask_ack)
|
|
desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
|
|
else {
|
|
desc->irq_data.chip->irq_mask(&desc->irq_data);
|
|
if (desc->irq_data.chip->irq_ack)
|
|
desc->irq_data.chip->irq_ack(&desc->irq_data);
|
|
}
|
|
irq_state_set_masked(desc);
|
|
}
|
|
|
|
void mask_irq(struct irq_desc *desc)
|
|
{
|
|
if (desc->irq_data.chip->irq_mask) {
|
|
desc->irq_data.chip->irq_mask(&desc->irq_data);
|
|
irq_state_set_masked(desc);
|
|
}
|
|
}
|
|
|
|
void unmask_irq(struct irq_desc *desc)
|
|
{
|
|
if (desc->irq_data.chip->irq_unmask) {
|
|
desc->irq_data.chip->irq_unmask(&desc->irq_data);
|
|
irq_state_clr_masked(desc);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* handle_nested_irq - Handle a nested irq from a irq thread
|
|
* @irq: the interrupt number
|
|
*
|
|
* Handle interrupts which are nested into a threaded interrupt
|
|
* handler. The handler function is called inside the calling
|
|
* threads context.
|
|
*/
|
|
void handle_nested_irq(unsigned int irq)
|
|
{
|
|
struct irq_desc *desc = irq_to_desc(irq);
|
|
struct irqaction *action;
|
|
irqreturn_t action_ret;
|
|
|
|
might_sleep();
|
|
|
|
raw_spin_lock_irq(&desc->lock);
|
|
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
action = desc->action;
|
|
if (unlikely(!action || irqd_irq_disabled(&desc->irq_data)))
|
|
goto out_unlock;
|
|
|
|
irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
|
|
raw_spin_unlock_irq(&desc->lock);
|
|
|
|
action_ret = action->thread_fn(action->irq, action->dev_id);
|
|
if (!noirqdebug)
|
|
note_interrupt(irq, desc, action_ret);
|
|
|
|
raw_spin_lock_irq(&desc->lock);
|
|
irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
|
|
|
|
out_unlock:
|
|
raw_spin_unlock_irq(&desc->lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(handle_nested_irq);
|
|
|
|
static bool irq_check_poll(struct irq_desc *desc)
|
|
{
|
|
if (!(desc->istate & IRQS_POLL_INPROGRESS))
|
|
return false;
|
|
return irq_wait_for_poll(desc);
|
|
}
|
|
|
|
/**
|
|
* handle_simple_irq - Simple and software-decoded IRQs.
|
|
* @irq: the interrupt number
|
|
* @desc: the interrupt description structure for this irq
|
|
*
|
|
* Simple interrupts are either sent from a demultiplexing interrupt
|
|
* handler or come from hardware, where no interrupt hardware control
|
|
* is necessary.
|
|
*
|
|
* Note: The caller is expected to handle the ack, clear, mask and
|
|
* unmask issues if necessary.
|
|
*/
|
|
void
|
|
handle_simple_irq(unsigned int irq, struct irq_desc *desc)
|
|
{
|
|
raw_spin_lock(&desc->lock);
|
|
|
|
if (unlikely(irqd_irq_inprogress(&desc->irq_data)))
|
|
if (!irq_check_poll(desc))
|
|
goto out_unlock;
|
|
|
|
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data)))
|
|
goto out_unlock;
|
|
|
|
handle_irq_event(desc);
|
|
|
|
out_unlock:
|
|
raw_spin_unlock(&desc->lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(handle_simple_irq);
|
|
|
|
/*
|
|
* Called unconditionally from handle_level_irq() and only for oneshot
|
|
* interrupts from handle_fasteoi_irq()
|
|
*/
|
|
static void cond_unmask_irq(struct irq_desc *desc)
|
|
{
|
|
/*
|
|
* We need to unmask in the following cases:
|
|
* - Standard level irq (IRQF_ONESHOT is not set)
|
|
* - Oneshot irq which did not wake the thread (caused by a
|
|
* spurious interrupt or a primary handler handling it
|
|
* completely).
|
|
*/
|
|
if (!irqd_irq_disabled(&desc->irq_data) &&
|
|
irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
|
|
unmask_irq(desc);
|
|
}
|
|
|
|
/**
|
|
* handle_level_irq - Level type irq handler
|
|
* @irq: the interrupt number
|
|
* @desc: the interrupt description structure for this irq
|
|
*
|
|
* Level type interrupts are active as long as the hardware line has
|
|
* the active level. This may require to mask the interrupt and unmask
|
|
* it after the associated handler has acknowledged the device, so the
|
|
* interrupt line is back to inactive.
|
|
*/
|
|
void
|
|
handle_level_irq(unsigned int irq, struct irq_desc *desc)
|
|
{
|
|
raw_spin_lock(&desc->lock);
|
|
mask_ack_irq(desc);
|
|
|
|
if (unlikely(irqd_irq_inprogress(&desc->irq_data)))
|
|
if (!irq_check_poll(desc))
|
|
goto out_unlock;
|
|
|
|
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
/*
|
|
* If its disabled or no action available
|
|
* keep it masked and get out of here
|
|
*/
|
|
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data)))
|
|
goto out_unlock;
|
|
|
|
handle_irq_event(desc);
|
|
|
|
cond_unmask_irq(desc);
|
|
|
|
out_unlock:
|
|
raw_spin_unlock(&desc->lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(handle_level_irq);
|
|
|
|
#ifdef CONFIG_IRQ_PREFLOW_FASTEOI
|
|
static inline void preflow_handler(struct irq_desc *desc)
|
|
{
|
|
if (desc->preflow_handler)
|
|
desc->preflow_handler(&desc->irq_data);
|
|
}
|
|
#else
|
|
static inline void preflow_handler(struct irq_desc *desc) { }
|
|
#endif
|
|
|
|
/**
|
|
* handle_fasteoi_irq - irq handler for transparent controllers
|
|
* @irq: the interrupt number
|
|
* @desc: the interrupt description structure for this irq
|
|
*
|
|
* Only a single callback will be issued to the chip: an ->eoi()
|
|
* call when the interrupt has been serviced. This enables support
|
|
* for modern forms of interrupt handlers, which handle the flow
|
|
* details in hardware, transparently.
|
|
*/
|
|
void
|
|
handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
|
|
{
|
|
raw_spin_lock(&desc->lock);
|
|
|
|
if (unlikely(irqd_irq_inprogress(&desc->irq_data)))
|
|
if (!irq_check_poll(desc))
|
|
goto out;
|
|
|
|
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
/*
|
|
* If its disabled or no action available
|
|
* then mask it and get out of here:
|
|
*/
|
|
if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
|
|
desc->istate |= IRQS_PENDING;
|
|
mask_irq(desc);
|
|
goto out;
|
|
}
|
|
|
|
if (desc->istate & IRQS_ONESHOT)
|
|
mask_irq(desc);
|
|
|
|
preflow_handler(desc);
|
|
handle_irq_event(desc);
|
|
|
|
if (desc->istate & IRQS_ONESHOT)
|
|
cond_unmask_irq(desc);
|
|
|
|
out_eoi:
|
|
desc->irq_data.chip->irq_eoi(&desc->irq_data);
|
|
out_unlock:
|
|
raw_spin_unlock(&desc->lock);
|
|
return;
|
|
out:
|
|
if (!(desc->irq_data.chip->flags & IRQCHIP_EOI_IF_HANDLED))
|
|
goto out_eoi;
|
|
goto out_unlock;
|
|
}
|
|
|
|
/**
|
|
* handle_edge_irq - edge type IRQ handler
|
|
* @irq: the interrupt number
|
|
* @desc: the interrupt description structure for this irq
|
|
*
|
|
* Interrupt occures on the falling and/or rising edge of a hardware
|
|
* signal. The occurrence is latched into the irq controller hardware
|
|
* and must be acked in order to be reenabled. After the ack another
|
|
* interrupt can happen on the same source even before the first one
|
|
* is handled by the associated event handler. If this happens it
|
|
* might be necessary to disable (mask) the interrupt depending on the
|
|
* controller hardware. This requires to reenable the interrupt inside
|
|
* of the loop which handles the interrupts which have arrived while
|
|
* the handler was running. If all pending interrupts are handled, the
|
|
* loop is left.
|
|
*/
|
|
void
|
|
handle_edge_irq(unsigned int irq, struct irq_desc *desc)
|
|
{
|
|
raw_spin_lock(&desc->lock);
|
|
|
|
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
|
|
/*
|
|
* If we're currently running this IRQ, or its disabled,
|
|
* we shouldn't process the IRQ. Mark it pending, handle
|
|
* the necessary masking and go out
|
|
*/
|
|
if (unlikely(irqd_irq_disabled(&desc->irq_data) ||
|
|
irqd_irq_inprogress(&desc->irq_data) || !desc->action)) {
|
|
if (!irq_check_poll(desc)) {
|
|
desc->istate |= IRQS_PENDING;
|
|
mask_ack_irq(desc);
|
|
goto out_unlock;
|
|
}
|
|
}
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
/* Start handling the irq */
|
|
desc->irq_data.chip->irq_ack(&desc->irq_data);
|
|
|
|
do {
|
|
if (unlikely(!desc->action)) {
|
|
mask_irq(desc);
|
|
goto out_unlock;
|
|
}
|
|
|
|
/*
|
|
* When another irq arrived while we were handling
|
|
* one, we could have masked the irq.
|
|
* Renable it, if it was not disabled in meantime.
|
|
*/
|
|
if (unlikely(desc->istate & IRQS_PENDING)) {
|
|
if (!irqd_irq_disabled(&desc->irq_data) &&
|
|
irqd_irq_masked(&desc->irq_data))
|
|
unmask_irq(desc);
|
|
}
|
|
|
|
handle_irq_event(desc);
|
|
|
|
} while ((desc->istate & IRQS_PENDING) &&
|
|
!irqd_irq_disabled(&desc->irq_data));
|
|
|
|
out_unlock:
|
|
raw_spin_unlock(&desc->lock);
|
|
}
|
|
|
|
#ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
|
|
/**
|
|
* handle_edge_eoi_irq - edge eoi type IRQ handler
|
|
* @irq: the interrupt number
|
|
* @desc: the interrupt description structure for this irq
|
|
*
|
|
* Similar as the above handle_edge_irq, but using eoi and w/o the
|
|
* mask/unmask logic.
|
|
*/
|
|
void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc)
|
|
{
|
|
struct irq_chip *chip = irq_desc_get_chip(desc);
|
|
|
|
raw_spin_lock(&desc->lock);
|
|
|
|
desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
|
|
/*
|
|
* If we're currently running this IRQ, or its disabled,
|
|
* we shouldn't process the IRQ. Mark it pending, handle
|
|
* the necessary masking and go out
|
|
*/
|
|
if (unlikely(irqd_irq_disabled(&desc->irq_data) ||
|
|
irqd_irq_inprogress(&desc->irq_data) || !desc->action)) {
|
|
if (!irq_check_poll(desc)) {
|
|
desc->istate |= IRQS_PENDING;
|
|
goto out_eoi;
|
|
}
|
|
}
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
do {
|
|
if (unlikely(!desc->action))
|
|
goto out_eoi;
|
|
|
|
handle_irq_event(desc);
|
|
|
|
} while ((desc->istate & IRQS_PENDING) &&
|
|
!irqd_irq_disabled(&desc->irq_data));
|
|
|
|
out_eoi:
|
|
chip->irq_eoi(&desc->irq_data);
|
|
raw_spin_unlock(&desc->lock);
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* handle_percpu_irq - Per CPU local irq handler
|
|
* @irq: the interrupt number
|
|
* @desc: the interrupt description structure for this irq
|
|
*
|
|
* Per CPU interrupts on SMP machines without locking requirements
|
|
*/
|
|
void
|
|
handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
|
|
{
|
|
struct irq_chip *chip = irq_desc_get_chip(desc);
|
|
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
if (chip->irq_ack)
|
|
chip->irq_ack(&desc->irq_data);
|
|
|
|
handle_irq_event_percpu(desc, desc->action);
|
|
|
|
if (chip->irq_eoi)
|
|
chip->irq_eoi(&desc->irq_data);
|
|
}
|
|
|
|
/**
|
|
* handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
|
|
* @irq: the interrupt number
|
|
* @desc: the interrupt description structure for this irq
|
|
*
|
|
* Per CPU interrupts on SMP machines without locking requirements. Same as
|
|
* handle_percpu_irq() above but with the following extras:
|
|
*
|
|
* action->percpu_dev_id is a pointer to percpu variables which
|
|
* contain the real device id for the cpu on which this handler is
|
|
* called
|
|
*/
|
|
void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc)
|
|
{
|
|
struct irq_chip *chip = irq_desc_get_chip(desc);
|
|
struct irqaction *action = desc->action;
|
|
void *dev_id = __this_cpu_ptr(action->percpu_dev_id);
|
|
irqreturn_t res;
|
|
|
|
kstat_incr_irqs_this_cpu(irq, desc);
|
|
|
|
if (chip->irq_ack)
|
|
chip->irq_ack(&desc->irq_data);
|
|
|
|
trace_irq_handler_entry(irq, action);
|
|
res = action->handler(irq, dev_id);
|
|
trace_irq_handler_exit(irq, action, res);
|
|
|
|
if (chip->irq_eoi)
|
|
chip->irq_eoi(&desc->irq_data);
|
|
}
|
|
|
|
void
|
|
__irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
|
|
const char *name)
|
|
{
|
|
unsigned long flags;
|
|
struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
|
|
|
|
if (!desc)
|
|
return;
|
|
|
|
if (!handle) {
|
|
handle = handle_bad_irq;
|
|
} else {
|
|
if (WARN_ON(desc->irq_data.chip == &no_irq_chip))
|
|
goto out;
|
|
}
|
|
|
|
/* Uninstall? */
|
|
if (handle == handle_bad_irq) {
|
|
if (desc->irq_data.chip != &no_irq_chip)
|
|
mask_ack_irq(desc);
|
|
irq_state_set_disabled(desc);
|
|
desc->depth = 1;
|
|
}
|
|
desc->handle_irq = handle;
|
|
desc->name = name;
|
|
|
|
if (handle != handle_bad_irq && is_chained) {
|
|
irq_settings_set_noprobe(desc);
|
|
irq_settings_set_norequest(desc);
|
|
irq_settings_set_nothread(desc);
|
|
irq_startup(desc, true);
|
|
}
|
|
out:
|
|
irq_put_desc_busunlock(desc, flags);
|
|
}
|
|
EXPORT_SYMBOL_GPL(__irq_set_handler);
|
|
|
|
void
|
|
irq_set_chip_and_handler_name(unsigned int irq, struct irq_chip *chip,
|
|
irq_flow_handler_t handle, const char *name)
|
|
{
|
|
irq_set_chip(irq, chip);
|
|
__irq_set_handler(irq, handle, 0, name);
|
|
}
|
|
|
|
void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
|
|
{
|
|
unsigned long flags;
|
|
struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
|
|
|
|
if (!desc)
|
|
return;
|
|
irq_settings_clr_and_set(desc, clr, set);
|
|
|
|
irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
|
|
IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
|
|
if (irq_settings_has_no_balance_set(desc))
|
|
irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
|
|
if (irq_settings_is_per_cpu(desc))
|
|
irqd_set(&desc->irq_data, IRQD_PER_CPU);
|
|
if (irq_settings_can_move_pcntxt(desc))
|
|
irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
|
|
if (irq_settings_is_level(desc))
|
|
irqd_set(&desc->irq_data, IRQD_LEVEL);
|
|
|
|
irqd_set(&desc->irq_data, irq_settings_get_trigger_mask(desc));
|
|
|
|
irq_put_desc_unlock(desc, flags);
|
|
}
|
|
EXPORT_SYMBOL_GPL(irq_modify_status);
|
|
|
|
/**
|
|
* irq_cpu_online - Invoke all irq_cpu_online functions.
|
|
*
|
|
* Iterate through all irqs and invoke the chip.irq_cpu_online()
|
|
* for each.
|
|
*/
|
|
void irq_cpu_online(void)
|
|
{
|
|
struct irq_desc *desc;
|
|
struct irq_chip *chip;
|
|
unsigned long flags;
|
|
unsigned int irq;
|
|
|
|
for_each_active_irq(irq) {
|
|
desc = irq_to_desc(irq);
|
|
if (!desc)
|
|
continue;
|
|
|
|
raw_spin_lock_irqsave(&desc->lock, flags);
|
|
|
|
chip = irq_data_get_irq_chip(&desc->irq_data);
|
|
if (chip && chip->irq_cpu_online &&
|
|
(!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
|
|
!irqd_irq_disabled(&desc->irq_data)))
|
|
chip->irq_cpu_online(&desc->irq_data);
|
|
|
|
raw_spin_unlock_irqrestore(&desc->lock, flags);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* irq_cpu_offline - Invoke all irq_cpu_offline functions.
|
|
*
|
|
* Iterate through all irqs and invoke the chip.irq_cpu_offline()
|
|
* for each.
|
|
*/
|
|
void irq_cpu_offline(void)
|
|
{
|
|
struct irq_desc *desc;
|
|
struct irq_chip *chip;
|
|
unsigned long flags;
|
|
unsigned int irq;
|
|
|
|
for_each_active_irq(irq) {
|
|
desc = irq_to_desc(irq);
|
|
if (!desc)
|
|
continue;
|
|
|
|
raw_spin_lock_irqsave(&desc->lock, flags);
|
|
|
|
chip = irq_data_get_irq_chip(&desc->irq_data);
|
|
if (chip && chip->irq_cpu_offline &&
|
|
(!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
|
|
!irqd_irq_disabled(&desc->irq_data)))
|
|
chip->irq_cpu_offline(&desc->irq_data);
|
|
|
|
raw_spin_unlock_irqrestore(&desc->lock, flags);
|
|
}
|
|
}
|