Root/kernel/irq_work.c

1/*
2 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
3 *
4 * Provides a framework for enqueueing and running callbacks from hardirq
5 * context. The enqueueing is NMI-safe.
6 */
7
8#include <linux/bug.h>
9#include <linux/kernel.h>
10#include <linux/export.h>
11#include <linux/irq_work.h>
12#include <linux/percpu.h>
13#include <linux/hardirq.h>
14#include <linux/irqflags.h>
15#include <linux/sched.h>
16#include <linux/tick.h>
17#include <linux/cpu.h>
18#include <linux/notifier.h>
19#include <asm/processor.h>
20
21
22static DEFINE_PER_CPU(struct llist_head, irq_work_list);
23static DEFINE_PER_CPU(int, irq_work_raised);
24
25/*
26 * Claim the entry so that no one else will poke at it.
27 */
28static bool irq_work_claim(struct irq_work *work)
29{
30    unsigned long flags, oflags, nflags;
31
32    /*
33     * Start with our best wish as a premise but only trust any
34     * flag value after cmpxchg() result.
35     */
36    flags = work->flags & ~IRQ_WORK_PENDING;
37    for (;;) {
38        nflags = flags | IRQ_WORK_FLAGS;
39        oflags = cmpxchg(&work->flags, flags, nflags);
40        if (oflags == flags)
41            break;
42        if (oflags & IRQ_WORK_PENDING)
43            return false;
44        flags = oflags;
45        cpu_relax();
46    }
47
48    return true;
49}
50
51void __weak arch_irq_work_raise(void)
52{
53    /*
54     * Lame architectures will get the timer tick callback
55     */
56}
57
58/*
59 * Enqueue the irq_work @entry unless it's already pending
60 * somewhere.
61 *
62 * Can be re-enqueued while the callback is still in progress.
63 */
64bool irq_work_queue(struct irq_work *work)
65{
66    /* Only queue if not already pending */
67    if (!irq_work_claim(work))
68        return false;
69
70    /* Queue the entry and raise the IPI if needed. */
71    preempt_disable();
72
73    llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
74
75    /*
76     * If the work is not "lazy" or the tick is stopped, raise the irq
77     * work interrupt (if supported by the arch), otherwise, just wait
78     * for the next tick.
79     */
80    if (!(work->flags & IRQ_WORK_LAZY) || tick_nohz_tick_stopped()) {
81        if (!this_cpu_cmpxchg(irq_work_raised, 0, 1))
82            arch_irq_work_raise();
83    }
84
85    preempt_enable();
86
87    return true;
88}
89EXPORT_SYMBOL_GPL(irq_work_queue);
90
91bool irq_work_needs_cpu(void)
92{
93    struct llist_head *this_list;
94
95    this_list = &__get_cpu_var(irq_work_list);
96    if (llist_empty(this_list))
97        return false;
98
99    /* All work should have been flushed before going offline */
100    WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
101
102    return true;
103}
104
105static void __irq_work_run(void)
106{
107    unsigned long flags;
108    struct irq_work *work;
109    struct llist_head *this_list;
110    struct llist_node *llnode;
111
112
113    /*
114     * Reset the "raised" state right before we check the list because
115     * an NMI may enqueue after we find the list empty from the runner.
116     */
117    __this_cpu_write(irq_work_raised, 0);
118    barrier();
119
120    this_list = &__get_cpu_var(irq_work_list);
121    if (llist_empty(this_list))
122        return;
123
124    BUG_ON(!irqs_disabled());
125
126    llnode = llist_del_all(this_list);
127    while (llnode != NULL) {
128        work = llist_entry(llnode, struct irq_work, llnode);
129
130        llnode = llist_next(llnode);
131
132        /*
133         * Clear the PENDING bit, after this point the @work
134         * can be re-used.
135         * Make it immediately visible so that other CPUs trying
136         * to claim that work don't rely on us to handle their data
137         * while we are in the middle of the func.
138         */
139        flags = work->flags & ~IRQ_WORK_PENDING;
140        xchg(&work->flags, flags);
141
142        work->func(work);
143        /*
144         * Clear the BUSY bit and return to the free state if
145         * no-one else claimed it meanwhile.
146         */
147        (void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
148    }
149}
150
151/*
152 * Run the irq_work entries on this cpu. Requires to be ran from hardirq
153 * context with local IRQs disabled.
154 */
155void irq_work_run(void)
156{
157    BUG_ON(!in_irq());
158    __irq_work_run();
159}
160EXPORT_SYMBOL_GPL(irq_work_run);
161
162/*
163 * Synchronize against the irq_work @entry, ensures the entry is not
164 * currently in use.
165 */
166void irq_work_sync(struct irq_work *work)
167{
168    WARN_ON_ONCE(irqs_disabled());
169
170    while (work->flags & IRQ_WORK_BUSY)
171        cpu_relax();
172}
173EXPORT_SYMBOL_GPL(irq_work_sync);
174
175#ifdef CONFIG_HOTPLUG_CPU
176static int irq_work_cpu_notify(struct notifier_block *self,
177                   unsigned long action, void *hcpu)
178{
179    long cpu = (long)hcpu;
180
181    switch (action) {
182    case CPU_DYING:
183        /* Called from stop_machine */
184        if (WARN_ON_ONCE(cpu != smp_processor_id()))
185            break;
186        __irq_work_run();
187        break;
188    default:
189        break;
190    }
191    return NOTIFY_OK;
192}
193
194static struct notifier_block cpu_notify;
195
196static __init int irq_work_init_cpu_notifier(void)
197{
198    cpu_notify.notifier_call = irq_work_cpu_notify;
199    cpu_notify.priority = 0;
200    register_cpu_notifier(&cpu_notify);
201    return 0;
202}
203device_initcall(irq_work_init_cpu_notifier);
204
205#endif /* CONFIG_HOTPLUG_CPU */
206

Archive Download this file



interactive