Root/kernel/softirq.c

1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 *
10 * Remote softirq infrastructure is by Jens Axboe.
11 */
12
13#include <linux/export.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
21#include <linux/freezer.h>
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
25#include <linux/smp.h>
26#include <linux/tick.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/irq.h>
30
31#include <asm/irq.h>
32/*
33   - No shared variables, all the data are CPU local.
34   - If a softirq needs serialization, let it serialize itself
35     by its own spinlocks.
36   - Even if softirq is serialized, only local cpu is marked for
37     execution. Hence, we get something sort of weak cpu binding.
38     Though it is still not clear, will it result in better locality
39     or will not.
40
41   Examples:
42   - NET RX softirq. It is multithreaded and does not require
43     any global serialization.
44   - NET TX softirq. It kicks software netdevice queues, hence
45     it is logically serialized per device, but this serialization
46     is invisible to common code.
47   - Tasklets: serialized wrt itself.
48 */
49
50#ifndef __ARCH_IRQ_STAT
51irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52EXPORT_SYMBOL(irq_stat);
53#endif
54
55static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56
57DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58
59char *softirq_to_name[NR_SOFTIRQS] = {
60    "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
61    "TASKLET", "SCHED", "HRTIMER", "RCU"
62};
63
64/*
65 * we cannot loop indefinitely here to avoid userspace starvation,
66 * but we also don't want to introduce a worst case 1/HZ latency
67 * to the pending events, so lets the scheduler to balance
68 * the softirq load for us.
69 */
70static void wakeup_softirqd(void)
71{
72    /* Interrupts are disabled: no need to stop preemption */
73    struct task_struct *tsk = __this_cpu_read(ksoftirqd);
74
75    if (tsk && tsk->state != TASK_RUNNING)
76        wake_up_process(tsk);
77}
78
79/*
80 * preempt_count and SOFTIRQ_OFFSET usage:
81 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
82 * softirq processing.
83 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
84 * on local_bh_disable or local_bh_enable.
85 * This lets us distinguish between whether we are currently processing
86 * softirq and whether we just have bh disabled.
87 */
88
89/*
90 * This one is for softirq.c-internal use,
91 * where hardirqs are disabled legitimately:
92 */
93#ifdef CONFIG_TRACE_IRQFLAGS
94static void __local_bh_disable(unsigned long ip, unsigned int cnt)
95{
96    unsigned long flags;
97
98    WARN_ON_ONCE(in_irq());
99
100    raw_local_irq_save(flags);
101    /*
102     * The preempt tracer hooks into add_preempt_count and will break
103     * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
104     * is set and before current->softirq_enabled is cleared.
105     * We must manually increment preempt_count here and manually
106     * call the trace_preempt_off later.
107     */
108    preempt_count() += cnt;
109    /*
110     * Were softirqs turned off above:
111     */
112    if (softirq_count() == cnt)
113        trace_softirqs_off(ip);
114    raw_local_irq_restore(flags);
115
116    if (preempt_count() == cnt)
117        trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
118}
119#else /* !CONFIG_TRACE_IRQFLAGS */
120static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
121{
122    add_preempt_count(cnt);
123    barrier();
124}
125#endif /* CONFIG_TRACE_IRQFLAGS */
126
127void local_bh_disable(void)
128{
129    __local_bh_disable((unsigned long)__builtin_return_address(0),
130                SOFTIRQ_DISABLE_OFFSET);
131}
132
133EXPORT_SYMBOL(local_bh_disable);
134
135static void __local_bh_enable(unsigned int cnt)
136{
137    WARN_ON_ONCE(in_irq());
138    WARN_ON_ONCE(!irqs_disabled());
139
140    if (softirq_count() == cnt)
141        trace_softirqs_on((unsigned long)__builtin_return_address(0));
142    sub_preempt_count(cnt);
143}
144
145/*
146 * Special-case - softirqs can safely be enabled in
147 * cond_resched_softirq(), or by __do_softirq(),
148 * without processing still-pending softirqs:
149 */
150void _local_bh_enable(void)
151{
152    __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
153}
154
155EXPORT_SYMBOL(_local_bh_enable);
156
157static inline void _local_bh_enable_ip(unsigned long ip)
158{
159    WARN_ON_ONCE(in_irq() || irqs_disabled());
160#ifdef CONFIG_TRACE_IRQFLAGS
161    local_irq_disable();
162#endif
163    /*
164     * Are softirqs going to be turned on now:
165     */
166    if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
167        trace_softirqs_on(ip);
168    /*
169     * Keep preemption disabled until we are done with
170     * softirq processing:
171      */
172    sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
173
174    if (unlikely(!in_interrupt() && local_softirq_pending()))
175        do_softirq();
176
177    dec_preempt_count();
178#ifdef CONFIG_TRACE_IRQFLAGS
179    local_irq_enable();
180#endif
181    preempt_check_resched();
182}
183
184void local_bh_enable(void)
185{
186    _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
187}
188EXPORT_SYMBOL(local_bh_enable);
189
190void local_bh_enable_ip(unsigned long ip)
191{
192    _local_bh_enable_ip(ip);
193}
194EXPORT_SYMBOL(local_bh_enable_ip);
195
196/*
197 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
198 * and we fall back to softirqd after that.
199 *
200 * This number has been established via experimentation.
201 * The two things to balance is latency against fairness -
202 * we want to handle softirqs as soon as possible, but they
203 * should not be able to lock up the box.
204 */
205#define MAX_SOFTIRQ_RESTART 10
206
207asmlinkage void __do_softirq(void)
208{
209    struct softirq_action *h;
210    __u32 pending;
211    int max_restart = MAX_SOFTIRQ_RESTART;
212    int cpu;
213
214    pending = local_softirq_pending();
215    account_system_vtime(current);
216
217    __local_bh_disable((unsigned long)__builtin_return_address(0),
218                SOFTIRQ_OFFSET);
219    lockdep_softirq_enter();
220
221    cpu = smp_processor_id();
222restart:
223    /* Reset the pending bitmask before enabling irqs */
224    set_softirq_pending(0);
225
226    local_irq_enable();
227
228    h = softirq_vec;
229
230    do {
231        if (pending & 1) {
232            unsigned int vec_nr = h - softirq_vec;
233            int prev_count = preempt_count();
234
235            kstat_incr_softirqs_this_cpu(vec_nr);
236
237            trace_softirq_entry(vec_nr);
238            h->action(h);
239            trace_softirq_exit(vec_nr);
240            if (unlikely(prev_count != preempt_count())) {
241                printk(KERN_ERR "huh, entered softirq %u %s %p"
242                       "with preempt_count %08x,"
243                       " exited with %08x?\n", vec_nr,
244                       softirq_to_name[vec_nr], h->action,
245                       prev_count, preempt_count());
246                preempt_count() = prev_count;
247            }
248
249            rcu_bh_qs(cpu);
250        }
251        h++;
252        pending >>= 1;
253    } while (pending);
254
255    local_irq_disable();
256
257    pending = local_softirq_pending();
258    if (pending && --max_restart)
259        goto restart;
260
261    if (pending)
262        wakeup_softirqd();
263
264    lockdep_softirq_exit();
265
266    account_system_vtime(current);
267    __local_bh_enable(SOFTIRQ_OFFSET);
268}
269
270#ifndef __ARCH_HAS_DO_SOFTIRQ
271
272asmlinkage void do_softirq(void)
273{
274    __u32 pending;
275    unsigned long flags;
276
277    if (in_interrupt())
278        return;
279
280    local_irq_save(flags);
281
282    pending = local_softirq_pending();
283
284    if (pending)
285        __do_softirq();
286
287    local_irq_restore(flags);
288}
289
290#endif
291
292/*
293 * Enter an interrupt context.
294 */
295void irq_enter(void)
296{
297    int cpu = smp_processor_id();
298
299    rcu_irq_enter();
300    if (is_idle_task(current) && !in_interrupt()) {
301        /*
302         * Prevent raise_softirq from needlessly waking up ksoftirqd
303         * here, as softirq will be serviced on return from interrupt.
304         */
305        local_bh_disable();
306        tick_check_idle(cpu);
307        _local_bh_enable();
308    }
309
310    __irq_enter();
311}
312
313static inline void invoke_softirq(void)
314{
315    if (!force_irqthreads) {
316#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
317        __do_softirq();
318#else
319        do_softirq();
320#endif
321    } else {
322        __local_bh_disable((unsigned long)__builtin_return_address(0),
323                SOFTIRQ_OFFSET);
324        wakeup_softirqd();
325        __local_bh_enable(SOFTIRQ_OFFSET);
326    }
327}
328
329/*
330 * Exit an interrupt context. Process softirqs if needed and possible:
331 */
332void irq_exit(void)
333{
334    account_system_vtime(current);
335    trace_hardirq_exit();
336    sub_preempt_count(IRQ_EXIT_OFFSET);
337    if (!in_interrupt() && local_softirq_pending())
338        invoke_softirq();
339
340#ifdef CONFIG_NO_HZ
341    /* Make sure that timer wheel updates are propagated */
342    if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
343        tick_nohz_irq_exit();
344#endif
345    rcu_irq_exit();
346    sched_preempt_enable_no_resched();
347}
348
349/*
350 * This function must run with irqs disabled!
351 */
352inline void raise_softirq_irqoff(unsigned int nr)
353{
354    __raise_softirq_irqoff(nr);
355
356    /*
357     * If we're in an interrupt or softirq, we're done
358     * (this also catches softirq-disabled code). We will
359     * actually run the softirq once we return from
360     * the irq or softirq.
361     *
362     * Otherwise we wake up ksoftirqd to make sure we
363     * schedule the softirq soon.
364     */
365    if (!in_interrupt())
366        wakeup_softirqd();
367}
368
369void raise_softirq(unsigned int nr)
370{
371    unsigned long flags;
372
373    local_irq_save(flags);
374    raise_softirq_irqoff(nr);
375    local_irq_restore(flags);
376}
377
378void __raise_softirq_irqoff(unsigned int nr)
379{
380    trace_softirq_raise(nr);
381    or_softirq_pending(1UL << nr);
382}
383
384void open_softirq(int nr, void (*action)(struct softirq_action *))
385{
386    softirq_vec[nr].action = action;
387}
388
389/*
390 * Tasklets
391 */
392struct tasklet_head
393{
394    struct tasklet_struct *head;
395    struct tasklet_struct **tail;
396};
397
398static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
399static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
400
401void __tasklet_schedule(struct tasklet_struct *t)
402{
403    unsigned long flags;
404
405    local_irq_save(flags);
406    t->next = NULL;
407    *__this_cpu_read(tasklet_vec.tail) = t;
408    __this_cpu_write(tasklet_vec.tail, &(t->next));
409    raise_softirq_irqoff(TASKLET_SOFTIRQ);
410    local_irq_restore(flags);
411}
412
413EXPORT_SYMBOL(__tasklet_schedule);
414
415void __tasklet_hi_schedule(struct tasklet_struct *t)
416{
417    unsigned long flags;
418
419    local_irq_save(flags);
420    t->next = NULL;
421    *__this_cpu_read(tasklet_hi_vec.tail) = t;
422    __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
423    raise_softirq_irqoff(HI_SOFTIRQ);
424    local_irq_restore(flags);
425}
426
427EXPORT_SYMBOL(__tasklet_hi_schedule);
428
429void __tasklet_hi_schedule_first(struct tasklet_struct *t)
430{
431    BUG_ON(!irqs_disabled());
432
433    t->next = __this_cpu_read(tasklet_hi_vec.head);
434    __this_cpu_write(tasklet_hi_vec.head, t);
435    __raise_softirq_irqoff(HI_SOFTIRQ);
436}
437
438EXPORT_SYMBOL(__tasklet_hi_schedule_first);
439
440static void tasklet_action(struct softirq_action *a)
441{
442    struct tasklet_struct *list;
443
444    local_irq_disable();
445    list = __this_cpu_read(tasklet_vec.head);
446    __this_cpu_write(tasklet_vec.head, NULL);
447    __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
448    local_irq_enable();
449
450    while (list) {
451        struct tasklet_struct *t = list;
452
453        list = list->next;
454
455        if (tasklet_trylock(t)) {
456            if (!atomic_read(&t->count)) {
457                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
458                    BUG();
459                t->func(t->data);
460                tasklet_unlock(t);
461                continue;
462            }
463            tasklet_unlock(t);
464        }
465
466        local_irq_disable();
467        t->next = NULL;
468        *__this_cpu_read(tasklet_vec.tail) = t;
469        __this_cpu_write(tasklet_vec.tail, &(t->next));
470        __raise_softirq_irqoff(TASKLET_SOFTIRQ);
471        local_irq_enable();
472    }
473}
474
475static void tasklet_hi_action(struct softirq_action *a)
476{
477    struct tasklet_struct *list;
478
479    local_irq_disable();
480    list = __this_cpu_read(tasklet_hi_vec.head);
481    __this_cpu_write(tasklet_hi_vec.head, NULL);
482    __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
483    local_irq_enable();
484
485    while (list) {
486        struct tasklet_struct *t = list;
487
488        list = list->next;
489
490        if (tasklet_trylock(t)) {
491            if (!atomic_read(&t->count)) {
492                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
493                    BUG();
494                t->func(t->data);
495                tasklet_unlock(t);
496                continue;
497            }
498            tasklet_unlock(t);
499        }
500
501        local_irq_disable();
502        t->next = NULL;
503        *__this_cpu_read(tasklet_hi_vec.tail) = t;
504        __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
505        __raise_softirq_irqoff(HI_SOFTIRQ);
506        local_irq_enable();
507    }
508}
509
510
511void tasklet_init(struct tasklet_struct *t,
512          void (*func)(unsigned long), unsigned long data)
513{
514    t->next = NULL;
515    t->state = 0;
516    atomic_set(&t->count, 0);
517    t->func = func;
518    t->data = data;
519}
520
521EXPORT_SYMBOL(tasklet_init);
522
523void tasklet_kill(struct tasklet_struct *t)
524{
525    if (in_interrupt())
526        printk("Attempt to kill tasklet from interrupt\n");
527
528    while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
529        do {
530            yield();
531        } while (test_bit(TASKLET_STATE_SCHED, &t->state));
532    }
533    tasklet_unlock_wait(t);
534    clear_bit(TASKLET_STATE_SCHED, &t->state);
535}
536
537EXPORT_SYMBOL(tasklet_kill);
538
539/*
540 * tasklet_hrtimer
541 */
542
543/*
544 * The trampoline is called when the hrtimer expires. It schedules a tasklet
545 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
546 * hrtimer callback, but from softirq context.
547 */
548static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
549{
550    struct tasklet_hrtimer *ttimer =
551        container_of(timer, struct tasklet_hrtimer, timer);
552
553    tasklet_hi_schedule(&ttimer->tasklet);
554    return HRTIMER_NORESTART;
555}
556
557/*
558 * Helper function which calls the hrtimer callback from
559 * tasklet/softirq context
560 */
561static void __tasklet_hrtimer_trampoline(unsigned long data)
562{
563    struct tasklet_hrtimer *ttimer = (void *)data;
564    enum hrtimer_restart restart;
565
566    restart = ttimer->function(&ttimer->timer);
567    if (restart != HRTIMER_NORESTART)
568        hrtimer_restart(&ttimer->timer);
569}
570
571/**
572 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
573 * @ttimer: tasklet_hrtimer which is initialized
574 * @function: hrtimer callback function which gets called from softirq context
575 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
576 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
577 */
578void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
579              enum hrtimer_restart (*function)(struct hrtimer *),
580              clockid_t which_clock, enum hrtimer_mode mode)
581{
582    hrtimer_init(&ttimer->timer, which_clock, mode);
583    ttimer->timer.function = __hrtimer_tasklet_trampoline;
584    tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
585             (unsigned long)ttimer);
586    ttimer->function = function;
587}
588EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
589
590/*
591 * Remote softirq bits
592 */
593
594DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
595EXPORT_PER_CPU_SYMBOL(softirq_work_list);
596
597static void __local_trigger(struct call_single_data *cp, int softirq)
598{
599    struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
600
601    list_add_tail(&cp->list, head);
602
603    /* Trigger the softirq only if the list was previously empty. */
604    if (head->next == &cp->list)
605        raise_softirq_irqoff(softirq);
606}
607
608#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
609static void remote_softirq_receive(void *data)
610{
611    struct call_single_data *cp = data;
612    unsigned long flags;
613    int softirq;
614
615    softirq = cp->priv;
616
617    local_irq_save(flags);
618    __local_trigger(cp, softirq);
619    local_irq_restore(flags);
620}
621
622static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
623{
624    if (cpu_online(cpu)) {
625        cp->func = remote_softirq_receive;
626        cp->info = cp;
627        cp->flags = 0;
628        cp->priv = softirq;
629
630        __smp_call_function_single(cpu, cp, 0);
631        return 0;
632    }
633    return 1;
634}
635#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
636static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
637{
638    return 1;
639}
640#endif
641
642/**
643 * __send_remote_softirq - try to schedule softirq work on a remote cpu
644 * @cp: private SMP call function data area
645 * @cpu: the remote cpu
646 * @this_cpu: the currently executing cpu
647 * @softirq: the softirq for the work
648 *
649 * Attempt to schedule softirq work on a remote cpu. If this cannot be
650 * done, the work is instead queued up on the local cpu.
651 *
652 * Interrupts must be disabled.
653 */
654void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
655{
656    if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
657        __local_trigger(cp, softirq);
658}
659EXPORT_SYMBOL(__send_remote_softirq);
660
661/**
662 * send_remote_softirq - try to schedule softirq work on a remote cpu
663 * @cp: private SMP call function data area
664 * @cpu: the remote cpu
665 * @softirq: the softirq for the work
666 *
667 * Like __send_remote_softirq except that disabling interrupts and
668 * computing the current cpu is done for the caller.
669 */
670void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
671{
672    unsigned long flags;
673    int this_cpu;
674
675    local_irq_save(flags);
676    this_cpu = smp_processor_id();
677    __send_remote_softirq(cp, cpu, this_cpu, softirq);
678    local_irq_restore(flags);
679}
680EXPORT_SYMBOL(send_remote_softirq);
681
682static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
683                           unsigned long action, void *hcpu)
684{
685    /*
686     * If a CPU goes away, splice its entries to the current CPU
687     * and trigger a run of the softirq
688     */
689    if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
690        int cpu = (unsigned long) hcpu;
691        int i;
692
693        local_irq_disable();
694        for (i = 0; i < NR_SOFTIRQS; i++) {
695            struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
696            struct list_head *local_head;
697
698            if (list_empty(head))
699                continue;
700
701            local_head = &__get_cpu_var(softirq_work_list[i]);
702            list_splice_init(head, local_head);
703            raise_softirq_irqoff(i);
704        }
705        local_irq_enable();
706    }
707
708    return NOTIFY_OK;
709}
710
711static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
712    .notifier_call = remote_softirq_cpu_notify,
713};
714
715void __init softirq_init(void)
716{
717    int cpu;
718
719    for_each_possible_cpu(cpu) {
720        int i;
721
722        per_cpu(tasklet_vec, cpu).tail =
723            &per_cpu(tasklet_vec, cpu).head;
724        per_cpu(tasklet_hi_vec, cpu).tail =
725            &per_cpu(tasklet_hi_vec, cpu).head;
726        for (i = 0; i < NR_SOFTIRQS; i++)
727            INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
728    }
729
730    register_hotcpu_notifier(&remote_softirq_cpu_notifier);
731
732    open_softirq(TASKLET_SOFTIRQ, tasklet_action);
733    open_softirq(HI_SOFTIRQ, tasklet_hi_action);
734}
735
736static int run_ksoftirqd(void * __bind_cpu)
737{
738    set_current_state(TASK_INTERRUPTIBLE);
739
740    while (!kthread_should_stop()) {
741        preempt_disable();
742        if (!local_softirq_pending()) {
743            schedule_preempt_disabled();
744        }
745
746        __set_current_state(TASK_RUNNING);
747
748        while (local_softirq_pending()) {
749            /* Preempt disable stops cpu going offline.
750               If already offline, we'll be on wrong CPU:
751               don't process */
752            if (cpu_is_offline((long)__bind_cpu))
753                goto wait_to_die;
754            local_irq_disable();
755            if (local_softirq_pending())
756                __do_softirq();
757            local_irq_enable();
758            sched_preempt_enable_no_resched();
759            cond_resched();
760            preempt_disable();
761            rcu_note_context_switch((long)__bind_cpu);
762        }
763        preempt_enable();
764        set_current_state(TASK_INTERRUPTIBLE);
765    }
766    __set_current_state(TASK_RUNNING);
767    return 0;
768
769wait_to_die:
770    preempt_enable();
771    /* Wait for kthread_stop */
772    set_current_state(TASK_INTERRUPTIBLE);
773    while (!kthread_should_stop()) {
774        schedule();
775        set_current_state(TASK_INTERRUPTIBLE);
776    }
777    __set_current_state(TASK_RUNNING);
778    return 0;
779}
780
781#ifdef CONFIG_HOTPLUG_CPU
782/*
783 * tasklet_kill_immediate is called to remove a tasklet which can already be
784 * scheduled for execution on @cpu.
785 *
786 * Unlike tasklet_kill, this function removes the tasklet
787 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
788 *
789 * When this function is called, @cpu must be in the CPU_DEAD state.
790 */
791void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
792{
793    struct tasklet_struct **i;
794
795    BUG_ON(cpu_online(cpu));
796    BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
797
798    if (!test_bit(TASKLET_STATE_SCHED, &t->state))
799        return;
800
801    /* CPU is dead, so no lock needed. */
802    for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
803        if (*i == t) {
804            *i = t->next;
805            /* If this was the tail element, move the tail ptr */
806            if (*i == NULL)
807                per_cpu(tasklet_vec, cpu).tail = i;
808            return;
809        }
810    }
811    BUG();
812}
813
814static void takeover_tasklets(unsigned int cpu)
815{
816    /* CPU is dead, so no lock needed. */
817    local_irq_disable();
818
819    /* Find end, append list for that CPU. */
820    if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
821        *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
822        this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
823        per_cpu(tasklet_vec, cpu).head = NULL;
824        per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
825    }
826    raise_softirq_irqoff(TASKLET_SOFTIRQ);
827
828    if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
829        *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
830        __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
831        per_cpu(tasklet_hi_vec, cpu).head = NULL;
832        per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
833    }
834    raise_softirq_irqoff(HI_SOFTIRQ);
835
836    local_irq_enable();
837}
838#endif /* CONFIG_HOTPLUG_CPU */
839
840static int __cpuinit cpu_callback(struct notifier_block *nfb,
841                  unsigned long action,
842                  void *hcpu)
843{
844    int hotcpu = (unsigned long)hcpu;
845    struct task_struct *p;
846
847    switch (action) {
848    case CPU_UP_PREPARE:
849    case CPU_UP_PREPARE_FROZEN:
850        p = kthread_create_on_node(run_ksoftirqd,
851                       hcpu,
852                       cpu_to_node(hotcpu),
853                       "ksoftirqd/%d", hotcpu);
854        if (IS_ERR(p)) {
855            printk("ksoftirqd for %i failed\n", hotcpu);
856            return notifier_from_errno(PTR_ERR(p));
857        }
858        kthread_bind(p, hotcpu);
859          per_cpu(ksoftirqd, hotcpu) = p;
860         break;
861    case CPU_ONLINE:
862    case CPU_ONLINE_FROZEN:
863        wake_up_process(per_cpu(ksoftirqd, hotcpu));
864        break;
865#ifdef CONFIG_HOTPLUG_CPU
866    case CPU_UP_CANCELED:
867    case CPU_UP_CANCELED_FROZEN:
868        if (!per_cpu(ksoftirqd, hotcpu))
869            break;
870        /* Unbind so it can run. Fall thru. */
871        kthread_bind(per_cpu(ksoftirqd, hotcpu),
872                 cpumask_any(cpu_online_mask));
873    case CPU_DEAD:
874    case CPU_DEAD_FROZEN: {
875        static const struct sched_param param = {
876            .sched_priority = MAX_RT_PRIO-1
877        };
878
879        p = per_cpu(ksoftirqd, hotcpu);
880        per_cpu(ksoftirqd, hotcpu) = NULL;
881        sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
882        kthread_stop(p);
883        takeover_tasklets(hotcpu);
884        break;
885    }
886#endif /* CONFIG_HOTPLUG_CPU */
887     }
888    return NOTIFY_OK;
889}
890
891static struct notifier_block __cpuinitdata cpu_nfb = {
892    .notifier_call = cpu_callback
893};
894
895static __init int spawn_ksoftirqd(void)
896{
897    void *cpu = (void *)(long)smp_processor_id();
898    int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
899
900    BUG_ON(err != NOTIFY_OK);
901    cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
902    register_cpu_notifier(&cpu_nfb);
903    return 0;
904}
905early_initcall(spawn_ksoftirqd);
906
907/*
908 * [ These __weak aliases are kept in a separate compilation unit, so that
909 * GCC does not inline them incorrectly. ]
910 */
911
912int __init __weak early_irq_init(void)
913{
914    return 0;
915}
916
917#ifdef CONFIG_GENERIC_HARDIRQS
918int __init __weak arch_probe_nr_irqs(void)
919{
920    return NR_IRQS_LEGACY;
921}
922
923int __init __weak arch_early_irq_init(void)
924{
925    return 0;
926}
927#endif
928

Archive Download this file



interactive