Root/kernel/softirq.c

1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 *
10 * Remote softirq infrastructure is by Jens Axboe.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
21#include <linux/freezer.h>
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
25#include <linux/smp.h>
26#include <linux/tick.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/irq.h>
30
31#include <asm/irq.h>
32/*
33   - No shared variables, all the data are CPU local.
34   - If a softirq needs serialization, let it serialize itself
35     by its own spinlocks.
36   - Even if softirq is serialized, only local cpu is marked for
37     execution. Hence, we get something sort of weak cpu binding.
38     Though it is still not clear, will it result in better locality
39     or will not.
40
41   Examples:
42   - NET RX softirq. It is multithreaded and does not require
43     any global serialization.
44   - NET TX softirq. It kicks software netdevice queues, hence
45     it is logically serialized per device, but this serialization
46     is invisible to common code.
47   - Tasklets: serialized wrt itself.
48 */
49
50#ifndef __ARCH_IRQ_STAT
51irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52EXPORT_SYMBOL(irq_stat);
53#endif
54
55static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56
57DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58
59char *softirq_to_name[NR_SOFTIRQS] = {
60    "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
61    "TASKLET", "SCHED", "HRTIMER", "RCU"
62};
63
64/*
65 * we cannot loop indefinitely here to avoid userspace starvation,
66 * but we also don't want to introduce a worst case 1/HZ latency
67 * to the pending events, so lets the scheduler to balance
68 * the softirq load for us.
69 */
70static void wakeup_softirqd(void)
71{
72    /* Interrupts are disabled: no need to stop preemption */
73    struct task_struct *tsk = __this_cpu_read(ksoftirqd);
74
75    if (tsk && tsk->state != TASK_RUNNING)
76        wake_up_process(tsk);
77}
78
79/*
80 * preempt_count and SOFTIRQ_OFFSET usage:
81 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
82 * softirq processing.
83 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
84 * on local_bh_disable or local_bh_enable.
85 * This lets us distinguish between whether we are currently processing
86 * softirq and whether we just have bh disabled.
87 */
88
89/*
90 * This one is for softirq.c-internal use,
91 * where hardirqs are disabled legitimately:
92 */
93#ifdef CONFIG_TRACE_IRQFLAGS
94static void __local_bh_disable(unsigned long ip, unsigned int cnt)
95{
96    unsigned long flags;
97
98    WARN_ON_ONCE(in_irq());
99
100    raw_local_irq_save(flags);
101    /*
102     * The preempt tracer hooks into add_preempt_count and will break
103     * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
104     * is set and before current->softirq_enabled is cleared.
105     * We must manually increment preempt_count here and manually
106     * call the trace_preempt_off later.
107     */
108    preempt_count() += cnt;
109    /*
110     * Were softirqs turned off above:
111     */
112    if (softirq_count() == cnt)
113        trace_softirqs_off(ip);
114    raw_local_irq_restore(flags);
115
116    if (preempt_count() == cnt)
117        trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
118}
119#else /* !CONFIG_TRACE_IRQFLAGS */
120static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
121{
122    add_preempt_count(cnt);
123    barrier();
124}
125#endif /* CONFIG_TRACE_IRQFLAGS */
126
127void local_bh_disable(void)
128{
129    __local_bh_disable((unsigned long)__builtin_return_address(0),
130                SOFTIRQ_DISABLE_OFFSET);
131}
132
133EXPORT_SYMBOL(local_bh_disable);
134
135static void __local_bh_enable(unsigned int cnt)
136{
137    WARN_ON_ONCE(in_irq());
138    WARN_ON_ONCE(!irqs_disabled());
139
140    if (softirq_count() == cnt)
141        trace_softirqs_on((unsigned long)__builtin_return_address(0));
142    sub_preempt_count(cnt);
143}
144
145/*
146 * Special-case - softirqs can safely be enabled in
147 * cond_resched_softirq(), or by __do_softirq(),
148 * without processing still-pending softirqs:
149 */
150void _local_bh_enable(void)
151{
152    __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
153}
154
155EXPORT_SYMBOL(_local_bh_enable);
156
157static inline void _local_bh_enable_ip(unsigned long ip)
158{
159    WARN_ON_ONCE(in_irq() || irqs_disabled());
160#ifdef CONFIG_TRACE_IRQFLAGS
161    local_irq_disable();
162#endif
163    /*
164     * Are softirqs going to be turned on now:
165     */
166    if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
167        trace_softirqs_on(ip);
168    /*
169     * Keep preemption disabled until we are done with
170     * softirq processing:
171      */
172    sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
173
174    if (unlikely(!in_interrupt() && local_softirq_pending()))
175        do_softirq();
176
177    dec_preempt_count();
178#ifdef CONFIG_TRACE_IRQFLAGS
179    local_irq_enable();
180#endif
181    preempt_check_resched();
182}
183
184void local_bh_enable(void)
185{
186    _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
187}
188EXPORT_SYMBOL(local_bh_enable);
189
190void local_bh_enable_ip(unsigned long ip)
191{
192    _local_bh_enable_ip(ip);
193}
194EXPORT_SYMBOL(local_bh_enable_ip);
195
196/*
197 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
198 * and we fall back to softirqd after that.
199 *
200 * This number has been established via experimentation.
201 * The two things to balance is latency against fairness -
202 * we want to handle softirqs as soon as possible, but they
203 * should not be able to lock up the box.
204 */
205#define MAX_SOFTIRQ_RESTART 10
206
207asmlinkage void __do_softirq(void)
208{
209    struct softirq_action *h;
210    __u32 pending;
211    int max_restart = MAX_SOFTIRQ_RESTART;
212    int cpu;
213
214    pending = local_softirq_pending();
215    account_system_vtime(current);
216
217    __local_bh_disable((unsigned long)__builtin_return_address(0),
218                SOFTIRQ_OFFSET);
219    lockdep_softirq_enter();
220
221    cpu = smp_processor_id();
222restart:
223    /* Reset the pending bitmask before enabling irqs */
224    set_softirq_pending(0);
225
226    local_irq_enable();
227
228    h = softirq_vec;
229
230    do {
231        if (pending & 1) {
232            unsigned int vec_nr = h - softirq_vec;
233            int prev_count = preempt_count();
234
235            kstat_incr_softirqs_this_cpu(vec_nr);
236
237            trace_softirq_entry(vec_nr);
238            h->action(h);
239            trace_softirq_exit(vec_nr);
240            if (unlikely(prev_count != preempt_count())) {
241                printk(KERN_ERR "huh, entered softirq %u %s %p"
242                       "with preempt_count %08x,"
243                       " exited with %08x?\n", vec_nr,
244                       softirq_to_name[vec_nr], h->action,
245                       prev_count, preempt_count());
246                preempt_count() = prev_count;
247            }
248
249            rcu_bh_qs(cpu);
250        }
251        h++;
252        pending >>= 1;
253    } while (pending);
254
255    local_irq_disable();
256
257    pending = local_softirq_pending();
258    if (pending && --max_restart)
259        goto restart;
260
261    if (pending)
262        wakeup_softirqd();
263
264    lockdep_softirq_exit();
265
266    account_system_vtime(current);
267    __local_bh_enable(SOFTIRQ_OFFSET);
268}
269
270#ifndef __ARCH_HAS_DO_SOFTIRQ
271
272asmlinkage void do_softirq(void)
273{
274    __u32 pending;
275    unsigned long flags;
276
277    if (in_interrupt())
278        return;
279
280    local_irq_save(flags);
281
282    pending = local_softirq_pending();
283
284    if (pending)
285        __do_softirq();
286
287    local_irq_restore(flags);
288}
289
290#endif
291
292/*
293 * Enter an interrupt context.
294 */
295void irq_enter(void)
296{
297    int cpu = smp_processor_id();
298
299    rcu_irq_enter();
300    if (idle_cpu(cpu) && !in_interrupt()) {
301        /*
302         * Prevent raise_softirq from needlessly waking up ksoftirqd
303         * here, as softirq will be serviced on return from interrupt.
304         */
305        local_bh_disable();
306        tick_check_idle(cpu);
307        _local_bh_enable();
308    }
309
310    __irq_enter();
311}
312
313#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
314static inline void invoke_softirq(void)
315{
316    if (!force_irqthreads)
317        __do_softirq();
318    else {
319        __local_bh_disable((unsigned long)__builtin_return_address(0),
320                SOFTIRQ_OFFSET);
321        wakeup_softirqd();
322        __local_bh_enable(SOFTIRQ_OFFSET);
323    }
324}
325#else
326static inline void invoke_softirq(void)
327{
328    if (!force_irqthreads)
329        do_softirq();
330    else {
331        __local_bh_disable((unsigned long)__builtin_return_address(0),
332                SOFTIRQ_OFFSET);
333        wakeup_softirqd();
334        __local_bh_enable(SOFTIRQ_OFFSET);
335    }
336}
337#endif
338
339/*
340 * Exit an interrupt context. Process softirqs if needed and possible:
341 */
342void irq_exit(void)
343{
344    account_system_vtime(current);
345    trace_hardirq_exit();
346    sub_preempt_count(IRQ_EXIT_OFFSET);
347    if (!in_interrupt() && local_softirq_pending())
348        invoke_softirq();
349
350    rcu_irq_exit();
351#ifdef CONFIG_NO_HZ
352    /* Make sure that timer wheel updates are propagated */
353    if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
354        tick_nohz_stop_sched_tick(0);
355#endif
356    preempt_enable_no_resched();
357}
358
359/*
360 * This function must run with irqs disabled!
361 */
362inline void raise_softirq_irqoff(unsigned int nr)
363{
364    __raise_softirq_irqoff(nr);
365
366    /*
367     * If we're in an interrupt or softirq, we're done
368     * (this also catches softirq-disabled code). We will
369     * actually run the softirq once we return from
370     * the irq or softirq.
371     *
372     * Otherwise we wake up ksoftirqd to make sure we
373     * schedule the softirq soon.
374     */
375    if (!in_interrupt())
376        wakeup_softirqd();
377}
378
379void raise_softirq(unsigned int nr)
380{
381    unsigned long flags;
382
383    local_irq_save(flags);
384    raise_softirq_irqoff(nr);
385    local_irq_restore(flags);
386}
387
388void open_softirq(int nr, void (*action)(struct softirq_action *))
389{
390    softirq_vec[nr].action = action;
391}
392
393/*
394 * Tasklets
395 */
396struct tasklet_head
397{
398    struct tasklet_struct *head;
399    struct tasklet_struct **tail;
400};
401
402static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
403static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
404
405void __tasklet_schedule(struct tasklet_struct *t)
406{
407    unsigned long flags;
408
409    local_irq_save(flags);
410    t->next = NULL;
411    *__this_cpu_read(tasklet_vec.tail) = t;
412    __this_cpu_write(tasklet_vec.tail, &(t->next));
413    raise_softirq_irqoff(TASKLET_SOFTIRQ);
414    local_irq_restore(flags);
415}
416
417EXPORT_SYMBOL(__tasklet_schedule);
418
419void __tasklet_hi_schedule(struct tasklet_struct *t)
420{
421    unsigned long flags;
422
423    local_irq_save(flags);
424    t->next = NULL;
425    *__this_cpu_read(tasklet_hi_vec.tail) = t;
426    __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
427    raise_softirq_irqoff(HI_SOFTIRQ);
428    local_irq_restore(flags);
429}
430
431EXPORT_SYMBOL(__tasklet_hi_schedule);
432
433void __tasklet_hi_schedule_first(struct tasklet_struct *t)
434{
435    BUG_ON(!irqs_disabled());
436
437    t->next = __this_cpu_read(tasklet_hi_vec.head);
438    __this_cpu_write(tasklet_hi_vec.head, t);
439    __raise_softirq_irqoff(HI_SOFTIRQ);
440}
441
442EXPORT_SYMBOL(__tasklet_hi_schedule_first);
443
444static void tasklet_action(struct softirq_action *a)
445{
446    struct tasklet_struct *list;
447
448    local_irq_disable();
449    list = __this_cpu_read(tasklet_vec.head);
450    __this_cpu_write(tasklet_vec.head, NULL);
451    __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
452    local_irq_enable();
453
454    while (list) {
455        struct tasklet_struct *t = list;
456
457        list = list->next;
458
459        if (tasklet_trylock(t)) {
460            if (!atomic_read(&t->count)) {
461                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
462                    BUG();
463                t->func(t->data);
464                tasklet_unlock(t);
465                continue;
466            }
467            tasklet_unlock(t);
468        }
469
470        local_irq_disable();
471        t->next = NULL;
472        *__this_cpu_read(tasklet_vec.tail) = t;
473        __this_cpu_write(tasklet_vec.tail, &(t->next));
474        __raise_softirq_irqoff(TASKLET_SOFTIRQ);
475        local_irq_enable();
476    }
477}
478
479static void tasklet_hi_action(struct softirq_action *a)
480{
481    struct tasklet_struct *list;
482
483    local_irq_disable();
484    list = __this_cpu_read(tasklet_hi_vec.head);
485    __this_cpu_write(tasklet_hi_vec.head, NULL);
486    __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
487    local_irq_enable();
488
489    while (list) {
490        struct tasklet_struct *t = list;
491
492        list = list->next;
493
494        if (tasklet_trylock(t)) {
495            if (!atomic_read(&t->count)) {
496                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
497                    BUG();
498                t->func(t->data);
499                tasklet_unlock(t);
500                continue;
501            }
502            tasklet_unlock(t);
503        }
504
505        local_irq_disable();
506        t->next = NULL;
507        *__this_cpu_read(tasklet_hi_vec.tail) = t;
508        __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
509        __raise_softirq_irqoff(HI_SOFTIRQ);
510        local_irq_enable();
511    }
512}
513
514
515void tasklet_init(struct tasklet_struct *t,
516          void (*func)(unsigned long), unsigned long data)
517{
518    t->next = NULL;
519    t->state = 0;
520    atomic_set(&t->count, 0);
521    t->func = func;
522    t->data = data;
523}
524
525EXPORT_SYMBOL(tasklet_init);
526
527void tasklet_kill(struct tasklet_struct *t)
528{
529    if (in_interrupt())
530        printk("Attempt to kill tasklet from interrupt\n");
531
532    while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
533        do {
534            yield();
535        } while (test_bit(TASKLET_STATE_SCHED, &t->state));
536    }
537    tasklet_unlock_wait(t);
538    clear_bit(TASKLET_STATE_SCHED, &t->state);
539}
540
541EXPORT_SYMBOL(tasklet_kill);
542
543/*
544 * tasklet_hrtimer
545 */
546
547/*
548 * The trampoline is called when the hrtimer expires. It schedules a tasklet
549 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
550 * hrtimer callback, but from softirq context.
551 */
552static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
553{
554    struct tasklet_hrtimer *ttimer =
555        container_of(timer, struct tasklet_hrtimer, timer);
556
557    tasklet_hi_schedule(&ttimer->tasklet);
558    return HRTIMER_NORESTART;
559}
560
561/*
562 * Helper function which calls the hrtimer callback from
563 * tasklet/softirq context
564 */
565static void __tasklet_hrtimer_trampoline(unsigned long data)
566{
567    struct tasklet_hrtimer *ttimer = (void *)data;
568    enum hrtimer_restart restart;
569
570    restart = ttimer->function(&ttimer->timer);
571    if (restart != HRTIMER_NORESTART)
572        hrtimer_restart(&ttimer->timer);
573}
574
575/**
576 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
577 * @ttimer: tasklet_hrtimer which is initialized
578 * @function: hrtimer callback function which gets called from softirq context
579 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
580 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
581 */
582void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
583              enum hrtimer_restart (*function)(struct hrtimer *),
584              clockid_t which_clock, enum hrtimer_mode mode)
585{
586    hrtimer_init(&ttimer->timer, which_clock, mode);
587    ttimer->timer.function = __hrtimer_tasklet_trampoline;
588    tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
589             (unsigned long)ttimer);
590    ttimer->function = function;
591}
592EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
593
594/*
595 * Remote softirq bits
596 */
597
598DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
599EXPORT_PER_CPU_SYMBOL(softirq_work_list);
600
601static void __local_trigger(struct call_single_data *cp, int softirq)
602{
603    struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
604
605    list_add_tail(&cp->list, head);
606
607    /* Trigger the softirq only if the list was previously empty. */
608    if (head->next == &cp->list)
609        raise_softirq_irqoff(softirq);
610}
611
612#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
613static void remote_softirq_receive(void *data)
614{
615    struct call_single_data *cp = data;
616    unsigned long flags;
617    int softirq;
618
619    softirq = cp->priv;
620
621    local_irq_save(flags);
622    __local_trigger(cp, softirq);
623    local_irq_restore(flags);
624}
625
626static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
627{
628    if (cpu_online(cpu)) {
629        cp->func = remote_softirq_receive;
630        cp->info = cp;
631        cp->flags = 0;
632        cp->priv = softirq;
633
634        __smp_call_function_single(cpu, cp, 0);
635        return 0;
636    }
637    return 1;
638}
639#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
640static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
641{
642    return 1;
643}
644#endif
645
646/**
647 * __send_remote_softirq - try to schedule softirq work on a remote cpu
648 * @cp: private SMP call function data area
649 * @cpu: the remote cpu
650 * @this_cpu: the currently executing cpu
651 * @softirq: the softirq for the work
652 *
653 * Attempt to schedule softirq work on a remote cpu. If this cannot be
654 * done, the work is instead queued up on the local cpu.
655 *
656 * Interrupts must be disabled.
657 */
658void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
659{
660    if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
661        __local_trigger(cp, softirq);
662}
663EXPORT_SYMBOL(__send_remote_softirq);
664
665/**
666 * send_remote_softirq - try to schedule softirq work on a remote cpu
667 * @cp: private SMP call function data area
668 * @cpu: the remote cpu
669 * @softirq: the softirq for the work
670 *
671 * Like __send_remote_softirq except that disabling interrupts and
672 * computing the current cpu is done for the caller.
673 */
674void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
675{
676    unsigned long flags;
677    int this_cpu;
678
679    local_irq_save(flags);
680    this_cpu = smp_processor_id();
681    __send_remote_softirq(cp, cpu, this_cpu, softirq);
682    local_irq_restore(flags);
683}
684EXPORT_SYMBOL(send_remote_softirq);
685
686static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
687                           unsigned long action, void *hcpu)
688{
689    /*
690     * If a CPU goes away, splice its entries to the current CPU
691     * and trigger a run of the softirq
692     */
693    if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
694        int cpu = (unsigned long) hcpu;
695        int i;
696
697        local_irq_disable();
698        for (i = 0; i < NR_SOFTIRQS; i++) {
699            struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
700            struct list_head *local_head;
701
702            if (list_empty(head))
703                continue;
704
705            local_head = &__get_cpu_var(softirq_work_list[i]);
706            list_splice_init(head, local_head);
707            raise_softirq_irqoff(i);
708        }
709        local_irq_enable();
710    }
711
712    return NOTIFY_OK;
713}
714
715static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
716    .notifier_call = remote_softirq_cpu_notify,
717};
718
719void __init softirq_init(void)
720{
721    int cpu;
722
723    for_each_possible_cpu(cpu) {
724        int i;
725
726        per_cpu(tasklet_vec, cpu).tail =
727            &per_cpu(tasklet_vec, cpu).head;
728        per_cpu(tasklet_hi_vec, cpu).tail =
729            &per_cpu(tasklet_hi_vec, cpu).head;
730        for (i = 0; i < NR_SOFTIRQS; i++)
731            INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
732    }
733
734    register_hotcpu_notifier(&remote_softirq_cpu_notifier);
735
736    open_softirq(TASKLET_SOFTIRQ, tasklet_action);
737    open_softirq(HI_SOFTIRQ, tasklet_hi_action);
738}
739
740static int run_ksoftirqd(void * __bind_cpu)
741{
742    set_current_state(TASK_INTERRUPTIBLE);
743
744    while (!kthread_should_stop()) {
745        preempt_disable();
746        if (!local_softirq_pending()) {
747            preempt_enable_no_resched();
748            schedule();
749            preempt_disable();
750        }
751
752        __set_current_state(TASK_RUNNING);
753
754        while (local_softirq_pending()) {
755            /* Preempt disable stops cpu going offline.
756               If already offline, we'll be on wrong CPU:
757               don't process */
758            if (cpu_is_offline((long)__bind_cpu))
759                goto wait_to_die;
760            local_irq_disable();
761            if (local_softirq_pending())
762                __do_softirq();
763            local_irq_enable();
764            preempt_enable_no_resched();
765            cond_resched();
766            preempt_disable();
767            rcu_note_context_switch((long)__bind_cpu);
768        }
769        preempt_enable();
770        set_current_state(TASK_INTERRUPTIBLE);
771    }
772    __set_current_state(TASK_RUNNING);
773    return 0;
774
775wait_to_die:
776    preempt_enable();
777    /* Wait for kthread_stop */
778    set_current_state(TASK_INTERRUPTIBLE);
779    while (!kthread_should_stop()) {
780        schedule();
781        set_current_state(TASK_INTERRUPTIBLE);
782    }
783    __set_current_state(TASK_RUNNING);
784    return 0;
785}
786
787#ifdef CONFIG_HOTPLUG_CPU
788/*
789 * tasklet_kill_immediate is called to remove a tasklet which can already be
790 * scheduled for execution on @cpu.
791 *
792 * Unlike tasklet_kill, this function removes the tasklet
793 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
794 *
795 * When this function is called, @cpu must be in the CPU_DEAD state.
796 */
797void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
798{
799    struct tasklet_struct **i;
800
801    BUG_ON(cpu_online(cpu));
802    BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
803
804    if (!test_bit(TASKLET_STATE_SCHED, &t->state))
805        return;
806
807    /* CPU is dead, so no lock needed. */
808    for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
809        if (*i == t) {
810            *i = t->next;
811            /* If this was the tail element, move the tail ptr */
812            if (*i == NULL)
813                per_cpu(tasklet_vec, cpu).tail = i;
814            return;
815        }
816    }
817    BUG();
818}
819
820static void takeover_tasklets(unsigned int cpu)
821{
822    /* CPU is dead, so no lock needed. */
823    local_irq_disable();
824
825    /* Find end, append list for that CPU. */
826    if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
827        *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
828        this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
829        per_cpu(tasklet_vec, cpu).head = NULL;
830        per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
831    }
832    raise_softirq_irqoff(TASKLET_SOFTIRQ);
833
834    if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
835        *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
836        __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
837        per_cpu(tasklet_hi_vec, cpu).head = NULL;
838        per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
839    }
840    raise_softirq_irqoff(HI_SOFTIRQ);
841
842    local_irq_enable();
843}
844#endif /* CONFIG_HOTPLUG_CPU */
845
846static int __cpuinit cpu_callback(struct notifier_block *nfb,
847                  unsigned long action,
848                  void *hcpu)
849{
850    int hotcpu = (unsigned long)hcpu;
851    struct task_struct *p;
852
853    switch (action) {
854    case CPU_UP_PREPARE:
855    case CPU_UP_PREPARE_FROZEN:
856        p = kthread_create_on_node(run_ksoftirqd,
857                       hcpu,
858                       cpu_to_node(hotcpu),
859                       "ksoftirqd/%d", hotcpu);
860        if (IS_ERR(p)) {
861            printk("ksoftirqd for %i failed\n", hotcpu);
862            return notifier_from_errno(PTR_ERR(p));
863        }
864        kthread_bind(p, hotcpu);
865          per_cpu(ksoftirqd, hotcpu) = p;
866         break;
867    case CPU_ONLINE:
868    case CPU_ONLINE_FROZEN:
869        wake_up_process(per_cpu(ksoftirqd, hotcpu));
870        break;
871#ifdef CONFIG_HOTPLUG_CPU
872    case CPU_UP_CANCELED:
873    case CPU_UP_CANCELED_FROZEN:
874        if (!per_cpu(ksoftirqd, hotcpu))
875            break;
876        /* Unbind so it can run. Fall thru. */
877        kthread_bind(per_cpu(ksoftirqd, hotcpu),
878                 cpumask_any(cpu_online_mask));
879    case CPU_DEAD:
880    case CPU_DEAD_FROZEN: {
881        static const struct sched_param param = {
882            .sched_priority = MAX_RT_PRIO-1
883        };
884
885        p = per_cpu(ksoftirqd, hotcpu);
886        per_cpu(ksoftirqd, hotcpu) = NULL;
887        sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
888        kthread_stop(p);
889        takeover_tasklets(hotcpu);
890        break;
891    }
892#endif /* CONFIG_HOTPLUG_CPU */
893     }
894    return NOTIFY_OK;
895}
896
897static struct notifier_block __cpuinitdata cpu_nfb = {
898    .notifier_call = cpu_callback
899};
900
901static __init int spawn_ksoftirqd(void)
902{
903    void *cpu = (void *)(long)smp_processor_id();
904    int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
905
906    BUG_ON(err != NOTIFY_OK);
907    cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
908    register_cpu_notifier(&cpu_nfb);
909    return 0;
910}
911early_initcall(spawn_ksoftirqd);
912
913/*
914 * [ These __weak aliases are kept in a separate compilation unit, so that
915 * GCC does not inline them incorrectly. ]
916 */
917
918int __init __weak early_irq_init(void)
919{
920    return 0;
921}
922
923#ifdef CONFIG_GENERIC_HARDIRQS
924int __init __weak arch_probe_nr_irqs(void)
925{
926    return NR_IRQS_LEGACY;
927}
928
929int __init __weak arch_early_irq_init(void)
930{
931    return 0;
932}
933#endif
934

Archive Download this file



interactive