Root/kernel/softirq.c

1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 *
10 * Remote softirq infrastructure is by Jens Axboe.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
21#include <linux/freezer.h>
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
25#include <linux/smp.h>
26#include <linux/tick.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/irq.h>
30
31#include <asm/irq.h>
32/*
33   - No shared variables, all the data are CPU local.
34   - If a softirq needs serialization, let it serialize itself
35     by its own spinlocks.
36   - Even if softirq is serialized, only local cpu is marked for
37     execution. Hence, we get something sort of weak cpu binding.
38     Though it is still not clear, will it result in better locality
39     or will not.
40
41   Examples:
42   - NET RX softirq. It is multithreaded and does not require
43     any global serialization.
44   - NET TX softirq. It kicks software netdevice queues, hence
45     it is logically serialized per device, but this serialization
46     is invisible to common code.
47   - Tasklets: serialized wrt itself.
48 */
49
50#ifndef __ARCH_IRQ_STAT
51irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52EXPORT_SYMBOL(irq_stat);
53#endif
54
55static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56
57static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58
59char *softirq_to_name[NR_SOFTIRQS] = {
60    "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
61    "TASKLET", "SCHED", "HRTIMER", "RCU"
62};
63
64/*
65 * we cannot loop indefinitely here to avoid userspace starvation,
66 * but we also don't want to introduce a worst case 1/HZ latency
67 * to the pending events, so lets the scheduler to balance
68 * the softirq load for us.
69 */
70static void wakeup_softirqd(void)
71{
72    /* Interrupts are disabled: no need to stop preemption */
73    struct task_struct *tsk = __this_cpu_read(ksoftirqd);
74
75    if (tsk && tsk->state != TASK_RUNNING)
76        wake_up_process(tsk);
77}
78
79/*
80 * preempt_count and SOFTIRQ_OFFSET usage:
81 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
82 * softirq processing.
83 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
84 * on local_bh_disable or local_bh_enable.
85 * This lets us distinguish between whether we are currently processing
86 * softirq and whether we just have bh disabled.
87 */
88
89/*
90 * This one is for softirq.c-internal use,
91 * where hardirqs are disabled legitimately:
92 */
93#ifdef CONFIG_TRACE_IRQFLAGS
94static void __local_bh_disable(unsigned long ip, unsigned int cnt)
95{
96    unsigned long flags;
97
98    WARN_ON_ONCE(in_irq());
99
100    raw_local_irq_save(flags);
101    /*
102     * The preempt tracer hooks into add_preempt_count and will break
103     * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
104     * is set and before current->softirq_enabled is cleared.
105     * We must manually increment preempt_count here and manually
106     * call the trace_preempt_off later.
107     */
108    preempt_count() += cnt;
109    /*
110     * Were softirqs turned off above:
111     */
112    if (softirq_count() == cnt)
113        trace_softirqs_off(ip);
114    raw_local_irq_restore(flags);
115
116    if (preempt_count() == cnt)
117        trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
118}
119#else /* !CONFIG_TRACE_IRQFLAGS */
120static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
121{
122    add_preempt_count(cnt);
123    barrier();
124}
125#endif /* CONFIG_TRACE_IRQFLAGS */
126
127void local_bh_disable(void)
128{
129    __local_bh_disable((unsigned long)__builtin_return_address(0),
130                SOFTIRQ_DISABLE_OFFSET);
131}
132
133EXPORT_SYMBOL(local_bh_disable);
134
135static void __local_bh_enable(unsigned int cnt)
136{
137    WARN_ON_ONCE(in_irq());
138    WARN_ON_ONCE(!irqs_disabled());
139
140    if (softirq_count() == cnt)
141        trace_softirqs_on((unsigned long)__builtin_return_address(0));
142    sub_preempt_count(cnt);
143}
144
145/*
146 * Special-case - softirqs can safely be enabled in
147 * cond_resched_softirq(), or by __do_softirq(),
148 * without processing still-pending softirqs:
149 */
150void _local_bh_enable(void)
151{
152    __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
153}
154
155EXPORT_SYMBOL(_local_bh_enable);
156
157static inline void _local_bh_enable_ip(unsigned long ip)
158{
159    WARN_ON_ONCE(in_irq() || irqs_disabled());
160#ifdef CONFIG_TRACE_IRQFLAGS
161    local_irq_disable();
162#endif
163    /*
164     * Are softirqs going to be turned on now:
165     */
166    if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
167        trace_softirqs_on(ip);
168    /*
169     * Keep preemption disabled until we are done with
170     * softirq processing:
171      */
172    sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
173
174    if (unlikely(!in_interrupt() && local_softirq_pending()))
175        do_softirq();
176
177    dec_preempt_count();
178#ifdef CONFIG_TRACE_IRQFLAGS
179    local_irq_enable();
180#endif
181    preempt_check_resched();
182}
183
184void local_bh_enable(void)
185{
186    _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
187}
188EXPORT_SYMBOL(local_bh_enable);
189
190void local_bh_enable_ip(unsigned long ip)
191{
192    _local_bh_enable_ip(ip);
193}
194EXPORT_SYMBOL(local_bh_enable_ip);
195
196/*
197 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
198 * and we fall back to softirqd after that.
199 *
200 * This number has been established via experimentation.
201 * The two things to balance is latency against fairness -
202 * we want to handle softirqs as soon as possible, but they
203 * should not be able to lock up the box.
204 */
205#define MAX_SOFTIRQ_RESTART 10
206
207asmlinkage void __do_softirq(void)
208{
209    struct softirq_action *h;
210    __u32 pending;
211    int max_restart = MAX_SOFTIRQ_RESTART;
212    int cpu;
213
214    pending = local_softirq_pending();
215    account_system_vtime(current);
216
217    __local_bh_disable((unsigned long)__builtin_return_address(0),
218                SOFTIRQ_OFFSET);
219    lockdep_softirq_enter();
220
221    cpu = smp_processor_id();
222restart:
223    /* Reset the pending bitmask before enabling irqs */
224    set_softirq_pending(0);
225
226    local_irq_enable();
227
228    h = softirq_vec;
229
230    do {
231        if (pending & 1) {
232            unsigned int vec_nr = h - softirq_vec;
233            int prev_count = preempt_count();
234
235            kstat_incr_softirqs_this_cpu(vec_nr);
236
237            trace_softirq_entry(vec_nr);
238            h->action(h);
239            trace_softirq_exit(vec_nr);
240            if (unlikely(prev_count != preempt_count())) {
241                printk(KERN_ERR "huh, entered softirq %u %s %p"
242                       "with preempt_count %08x,"
243                       " exited with %08x?\n", vec_nr,
244                       softirq_to_name[vec_nr], h->action,
245                       prev_count, preempt_count());
246                preempt_count() = prev_count;
247            }
248
249            rcu_bh_qs(cpu);
250        }
251        h++;
252        pending >>= 1;
253    } while (pending);
254
255    local_irq_disable();
256
257    pending = local_softirq_pending();
258    if (pending && --max_restart)
259        goto restart;
260
261    if (pending)
262        wakeup_softirqd();
263
264    lockdep_softirq_exit();
265
266    account_system_vtime(current);
267    __local_bh_enable(SOFTIRQ_OFFSET);
268}
269
270#ifndef __ARCH_HAS_DO_SOFTIRQ
271
272asmlinkage void do_softirq(void)
273{
274    __u32 pending;
275    unsigned long flags;
276
277    if (in_interrupt())
278        return;
279
280    local_irq_save(flags);
281
282    pending = local_softirq_pending();
283
284    if (pending)
285        __do_softirq();
286
287    local_irq_restore(flags);
288}
289
290#endif
291
292/*
293 * Enter an interrupt context.
294 */
295void irq_enter(void)
296{
297    int cpu = smp_processor_id();
298
299    rcu_irq_enter();
300    if (idle_cpu(cpu) && !in_interrupt()) {
301        /*
302         * Prevent raise_softirq from needlessly waking up ksoftirqd
303         * here, as softirq will be serviced on return from interrupt.
304         */
305        local_bh_disable();
306        tick_check_idle(cpu);
307        _local_bh_enable();
308    }
309
310    __irq_enter();
311}
312
313#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
314# define invoke_softirq() __do_softirq()
315#else
316# define invoke_softirq() do_softirq()
317#endif
318
319/*
320 * Exit an interrupt context. Process softirqs if needed and possible:
321 */
322void irq_exit(void)
323{
324    account_system_vtime(current);
325    trace_hardirq_exit();
326    sub_preempt_count(IRQ_EXIT_OFFSET);
327    if (!in_interrupt() && local_softirq_pending())
328        invoke_softirq();
329
330    rcu_irq_exit();
331#ifdef CONFIG_NO_HZ
332    /* Make sure that timer wheel updates are propagated */
333    if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
334        tick_nohz_stop_sched_tick(0);
335#endif
336    preempt_enable_no_resched();
337}
338
339/*
340 * This function must run with irqs disabled!
341 */
342inline void raise_softirq_irqoff(unsigned int nr)
343{
344    __raise_softirq_irqoff(nr);
345
346    /*
347     * If we're in an interrupt or softirq, we're done
348     * (this also catches softirq-disabled code). We will
349     * actually run the softirq once we return from
350     * the irq or softirq.
351     *
352     * Otherwise we wake up ksoftirqd to make sure we
353     * schedule the softirq soon.
354     */
355    if (!in_interrupt())
356        wakeup_softirqd();
357}
358
359void raise_softirq(unsigned int nr)
360{
361    unsigned long flags;
362
363    local_irq_save(flags);
364    raise_softirq_irqoff(nr);
365    local_irq_restore(flags);
366}
367
368void open_softirq(int nr, void (*action)(struct softirq_action *))
369{
370    softirq_vec[nr].action = action;
371}
372
373/*
374 * Tasklets
375 */
376struct tasklet_head
377{
378    struct tasklet_struct *head;
379    struct tasklet_struct **tail;
380};
381
382static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
383static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
384
385void __tasklet_schedule(struct tasklet_struct *t)
386{
387    unsigned long flags;
388
389    local_irq_save(flags);
390    t->next = NULL;
391    *__this_cpu_read(tasklet_vec.tail) = t;
392    __this_cpu_write(tasklet_vec.tail, &(t->next));
393    raise_softirq_irqoff(TASKLET_SOFTIRQ);
394    local_irq_restore(flags);
395}
396
397EXPORT_SYMBOL(__tasklet_schedule);
398
399void __tasklet_hi_schedule(struct tasklet_struct *t)
400{
401    unsigned long flags;
402
403    local_irq_save(flags);
404    t->next = NULL;
405    *__this_cpu_read(tasklet_hi_vec.tail) = t;
406    __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
407    raise_softirq_irqoff(HI_SOFTIRQ);
408    local_irq_restore(flags);
409}
410
411EXPORT_SYMBOL(__tasklet_hi_schedule);
412
413void __tasklet_hi_schedule_first(struct tasklet_struct *t)
414{
415    BUG_ON(!irqs_disabled());
416
417    t->next = __this_cpu_read(tasklet_hi_vec.head);
418    __this_cpu_write(tasklet_hi_vec.head, t);
419    __raise_softirq_irqoff(HI_SOFTIRQ);
420}
421
422EXPORT_SYMBOL(__tasklet_hi_schedule_first);
423
424static void tasklet_action(struct softirq_action *a)
425{
426    struct tasklet_struct *list;
427
428    local_irq_disable();
429    list = __this_cpu_read(tasklet_vec.head);
430    __this_cpu_write(tasklet_vec.head, NULL);
431    __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
432    local_irq_enable();
433
434    while (list) {
435        struct tasklet_struct *t = list;
436
437        list = list->next;
438
439        if (tasklet_trylock(t)) {
440            if (!atomic_read(&t->count)) {
441                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
442                    BUG();
443                t->func(t->data);
444                tasklet_unlock(t);
445                continue;
446            }
447            tasklet_unlock(t);
448        }
449
450        local_irq_disable();
451        t->next = NULL;
452        *__this_cpu_read(tasklet_vec.tail) = t;
453        __this_cpu_write(tasklet_vec.tail, &(t->next));
454        __raise_softirq_irqoff(TASKLET_SOFTIRQ);
455        local_irq_enable();
456    }
457}
458
459static void tasklet_hi_action(struct softirq_action *a)
460{
461    struct tasklet_struct *list;
462
463    local_irq_disable();
464    list = __this_cpu_read(tasklet_hi_vec.head);
465    __this_cpu_write(tasklet_hi_vec.head, NULL);
466    __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
467    local_irq_enable();
468
469    while (list) {
470        struct tasklet_struct *t = list;
471
472        list = list->next;
473
474        if (tasklet_trylock(t)) {
475            if (!atomic_read(&t->count)) {
476                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
477                    BUG();
478                t->func(t->data);
479                tasklet_unlock(t);
480                continue;
481            }
482            tasklet_unlock(t);
483        }
484
485        local_irq_disable();
486        t->next = NULL;
487        *__this_cpu_read(tasklet_hi_vec.tail) = t;
488        __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
489        __raise_softirq_irqoff(HI_SOFTIRQ);
490        local_irq_enable();
491    }
492}
493
494
495void tasklet_init(struct tasklet_struct *t,
496          void (*func)(unsigned long), unsigned long data)
497{
498    t->next = NULL;
499    t->state = 0;
500    atomic_set(&t->count, 0);
501    t->func = func;
502    t->data = data;
503}
504
505EXPORT_SYMBOL(tasklet_init);
506
507void tasklet_kill(struct tasklet_struct *t)
508{
509    if (in_interrupt())
510        printk("Attempt to kill tasklet from interrupt\n");
511
512    while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
513        do {
514            yield();
515        } while (test_bit(TASKLET_STATE_SCHED, &t->state));
516    }
517    tasklet_unlock_wait(t);
518    clear_bit(TASKLET_STATE_SCHED, &t->state);
519}
520
521EXPORT_SYMBOL(tasklet_kill);
522
523/*
524 * tasklet_hrtimer
525 */
526
527/*
528 * The trampoline is called when the hrtimer expires. It schedules a tasklet
529 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
530 * hrtimer callback, but from softirq context.
531 */
532static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
533{
534    struct tasklet_hrtimer *ttimer =
535        container_of(timer, struct tasklet_hrtimer, timer);
536
537    tasklet_hi_schedule(&ttimer->tasklet);
538    return HRTIMER_NORESTART;
539}
540
541/*
542 * Helper function which calls the hrtimer callback from
543 * tasklet/softirq context
544 */
545static void __tasklet_hrtimer_trampoline(unsigned long data)
546{
547    struct tasklet_hrtimer *ttimer = (void *)data;
548    enum hrtimer_restart restart;
549
550    restart = ttimer->function(&ttimer->timer);
551    if (restart != HRTIMER_NORESTART)
552        hrtimer_restart(&ttimer->timer);
553}
554
555/**
556 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
557 * @ttimer: tasklet_hrtimer which is initialized
558 * @function: hrtimer callback funtion which gets called from softirq context
559 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
560 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
561 */
562void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
563              enum hrtimer_restart (*function)(struct hrtimer *),
564              clockid_t which_clock, enum hrtimer_mode mode)
565{
566    hrtimer_init(&ttimer->timer, which_clock, mode);
567    ttimer->timer.function = __hrtimer_tasklet_trampoline;
568    tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
569             (unsigned long)ttimer);
570    ttimer->function = function;
571}
572EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
573
574/*
575 * Remote softirq bits
576 */
577
578DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
579EXPORT_PER_CPU_SYMBOL(softirq_work_list);
580
581static void __local_trigger(struct call_single_data *cp, int softirq)
582{
583    struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
584
585    list_add_tail(&cp->list, head);
586
587    /* Trigger the softirq only if the list was previously empty. */
588    if (head->next == &cp->list)
589        raise_softirq_irqoff(softirq);
590}
591
592#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
593static void remote_softirq_receive(void *data)
594{
595    struct call_single_data *cp = data;
596    unsigned long flags;
597    int softirq;
598
599    softirq = cp->priv;
600
601    local_irq_save(flags);
602    __local_trigger(cp, softirq);
603    local_irq_restore(flags);
604}
605
606static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
607{
608    if (cpu_online(cpu)) {
609        cp->func = remote_softirq_receive;
610        cp->info = cp;
611        cp->flags = 0;
612        cp->priv = softirq;
613
614        __smp_call_function_single(cpu, cp, 0);
615        return 0;
616    }
617    return 1;
618}
619#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
620static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
621{
622    return 1;
623}
624#endif
625
626/**
627 * __send_remote_softirq - try to schedule softirq work on a remote cpu
628 * @cp: private SMP call function data area
629 * @cpu: the remote cpu
630 * @this_cpu: the currently executing cpu
631 * @softirq: the softirq for the work
632 *
633 * Attempt to schedule softirq work on a remote cpu. If this cannot be
634 * done, the work is instead queued up on the local cpu.
635 *
636 * Interrupts must be disabled.
637 */
638void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
639{
640    if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
641        __local_trigger(cp, softirq);
642}
643EXPORT_SYMBOL(__send_remote_softirq);
644
645/**
646 * send_remote_softirq - try to schedule softirq work on a remote cpu
647 * @cp: private SMP call function data area
648 * @cpu: the remote cpu
649 * @softirq: the softirq for the work
650 *
651 * Like __send_remote_softirq except that disabling interrupts and
652 * computing the current cpu is done for the caller.
653 */
654void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
655{
656    unsigned long flags;
657    int this_cpu;
658
659    local_irq_save(flags);
660    this_cpu = smp_processor_id();
661    __send_remote_softirq(cp, cpu, this_cpu, softirq);
662    local_irq_restore(flags);
663}
664EXPORT_SYMBOL(send_remote_softirq);
665
666static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
667                           unsigned long action, void *hcpu)
668{
669    /*
670     * If a CPU goes away, splice its entries to the current CPU
671     * and trigger a run of the softirq
672     */
673    if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
674        int cpu = (unsigned long) hcpu;
675        int i;
676
677        local_irq_disable();
678        for (i = 0; i < NR_SOFTIRQS; i++) {
679            struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
680            struct list_head *local_head;
681
682            if (list_empty(head))
683                continue;
684
685            local_head = &__get_cpu_var(softirq_work_list[i]);
686            list_splice_init(head, local_head);
687            raise_softirq_irqoff(i);
688        }
689        local_irq_enable();
690    }
691
692    return NOTIFY_OK;
693}
694
695static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
696    .notifier_call = remote_softirq_cpu_notify,
697};
698
699void __init softirq_init(void)
700{
701    int cpu;
702
703    for_each_possible_cpu(cpu) {
704        int i;
705
706        per_cpu(tasklet_vec, cpu).tail =
707            &per_cpu(tasklet_vec, cpu).head;
708        per_cpu(tasklet_hi_vec, cpu).tail =
709            &per_cpu(tasklet_hi_vec, cpu).head;
710        for (i = 0; i < NR_SOFTIRQS; i++)
711            INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
712    }
713
714    register_hotcpu_notifier(&remote_softirq_cpu_notifier);
715
716    open_softirq(TASKLET_SOFTIRQ, tasklet_action);
717    open_softirq(HI_SOFTIRQ, tasklet_hi_action);
718}
719
720static int run_ksoftirqd(void * __bind_cpu)
721{
722    set_current_state(TASK_INTERRUPTIBLE);
723
724    current->flags |= PF_KSOFTIRQD;
725    while (!kthread_should_stop()) {
726        preempt_disable();
727        if (!local_softirq_pending()) {
728            preempt_enable_no_resched();
729            schedule();
730            preempt_disable();
731        }
732
733        __set_current_state(TASK_RUNNING);
734
735        while (local_softirq_pending()) {
736            /* Preempt disable stops cpu going offline.
737               If already offline, we'll be on wrong CPU:
738               don't process */
739            if (cpu_is_offline((long)__bind_cpu))
740                goto wait_to_die;
741            do_softirq();
742            preempt_enable_no_resched();
743            cond_resched();
744            preempt_disable();
745            rcu_note_context_switch((long)__bind_cpu);
746        }
747        preempt_enable();
748        set_current_state(TASK_INTERRUPTIBLE);
749    }
750    __set_current_state(TASK_RUNNING);
751    return 0;
752
753wait_to_die:
754    preempt_enable();
755    /* Wait for kthread_stop */
756    set_current_state(TASK_INTERRUPTIBLE);
757    while (!kthread_should_stop()) {
758        schedule();
759        set_current_state(TASK_INTERRUPTIBLE);
760    }
761    __set_current_state(TASK_RUNNING);
762    return 0;
763}
764
765#ifdef CONFIG_HOTPLUG_CPU
766/*
767 * tasklet_kill_immediate is called to remove a tasklet which can already be
768 * scheduled for execution on @cpu.
769 *
770 * Unlike tasklet_kill, this function removes the tasklet
771 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
772 *
773 * When this function is called, @cpu must be in the CPU_DEAD state.
774 */
775void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
776{
777    struct tasklet_struct **i;
778
779    BUG_ON(cpu_online(cpu));
780    BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
781
782    if (!test_bit(TASKLET_STATE_SCHED, &t->state))
783        return;
784
785    /* CPU is dead, so no lock needed. */
786    for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
787        if (*i == t) {
788            *i = t->next;
789            /* If this was the tail element, move the tail ptr */
790            if (*i == NULL)
791                per_cpu(tasklet_vec, cpu).tail = i;
792            return;
793        }
794    }
795    BUG();
796}
797
798static void takeover_tasklets(unsigned int cpu)
799{
800    /* CPU is dead, so no lock needed. */
801    local_irq_disable();
802
803    /* Find end, append list for that CPU. */
804    if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
805        *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
806        this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
807        per_cpu(tasklet_vec, cpu).head = NULL;
808        per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
809    }
810    raise_softirq_irqoff(TASKLET_SOFTIRQ);
811
812    if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
813        *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
814        __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
815        per_cpu(tasklet_hi_vec, cpu).head = NULL;
816        per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
817    }
818    raise_softirq_irqoff(HI_SOFTIRQ);
819
820    local_irq_enable();
821}
822#endif /* CONFIG_HOTPLUG_CPU */
823
824static int __cpuinit cpu_callback(struct notifier_block *nfb,
825                  unsigned long action,
826                  void *hcpu)
827{
828    int hotcpu = (unsigned long)hcpu;
829    struct task_struct *p;
830
831    switch (action) {
832    case CPU_UP_PREPARE:
833    case CPU_UP_PREPARE_FROZEN:
834        p = kthread_create(run_ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
835        if (IS_ERR(p)) {
836            printk("ksoftirqd for %i failed\n", hotcpu);
837            return notifier_from_errno(PTR_ERR(p));
838        }
839        kthread_bind(p, hotcpu);
840          per_cpu(ksoftirqd, hotcpu) = p;
841         break;
842    case CPU_ONLINE:
843    case CPU_ONLINE_FROZEN:
844        wake_up_process(per_cpu(ksoftirqd, hotcpu));
845        break;
846#ifdef CONFIG_HOTPLUG_CPU
847    case CPU_UP_CANCELED:
848    case CPU_UP_CANCELED_FROZEN:
849        if (!per_cpu(ksoftirqd, hotcpu))
850            break;
851        /* Unbind so it can run. Fall thru. */
852        kthread_bind(per_cpu(ksoftirqd, hotcpu),
853                 cpumask_any(cpu_online_mask));
854    case CPU_DEAD:
855    case CPU_DEAD_FROZEN: {
856        static const struct sched_param param = {
857            .sched_priority = MAX_RT_PRIO-1
858        };
859
860        p = per_cpu(ksoftirqd, hotcpu);
861        per_cpu(ksoftirqd, hotcpu) = NULL;
862        sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
863        kthread_stop(p);
864        takeover_tasklets(hotcpu);
865        break;
866    }
867#endif /* CONFIG_HOTPLUG_CPU */
868     }
869    return NOTIFY_OK;
870}
871
872static struct notifier_block __cpuinitdata cpu_nfb = {
873    .notifier_call = cpu_callback
874};
875
876static __init int spawn_ksoftirqd(void)
877{
878    void *cpu = (void *)(long)smp_processor_id();
879    int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
880
881    BUG_ON(err != NOTIFY_OK);
882    cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
883    register_cpu_notifier(&cpu_nfb);
884    return 0;
885}
886early_initcall(spawn_ksoftirqd);
887
888/*
889 * [ These __weak aliases are kept in a separate compilation unit, so that
890 * GCC does not inline them incorrectly. ]
891 */
892
893int __init __weak early_irq_init(void)
894{
895    return 0;
896}
897
898#ifdef CONFIG_GENERIC_HARDIRQS
899int __init __weak arch_probe_nr_irqs(void)
900{
901    return NR_IRQS_LEGACY;
902}
903
904int __init __weak arch_early_irq_init(void)
905{
906    return 0;
907}
908#endif
909

Archive Download this file



interactive