Root/kernel/rcutiny.c

1/*
2 * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright IBM Corporation, 2008
19 *
20 * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
21 *
22 * For detailed explanation of Read-Copy Update mechanism see -
23 * Documentation/RCU
24 */
25#include <linux/moduleparam.h>
26#include <linux/completion.h>
27#include <linux/interrupt.h>
28#include <linux/notifier.h>
29#include <linux/rcupdate.h>
30#include <linux/kernel.h>
31#include <linux/module.h>
32#include <linux/mutex.h>
33#include <linux/sched.h>
34#include <linux/types.h>
35#include <linux/init.h>
36#include <linux/time.h>
37#include <linux/cpu.h>
38
39/* Global control variables for rcupdate callback mechanism. */
40struct rcu_ctrlblk {
41    struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */
42    struct rcu_head **donetail; /* ->next pointer of last "done" CB. */
43    struct rcu_head **curtail; /* ->next pointer of last CB. */
44};
45
46/* Definition for rcupdate control block. */
47static struct rcu_ctrlblk rcu_sched_ctrlblk = {
48    .donetail = &rcu_sched_ctrlblk.rcucblist,
49    .curtail = &rcu_sched_ctrlblk.rcucblist,
50};
51
52static struct rcu_ctrlblk rcu_bh_ctrlblk = {
53    .donetail = &rcu_bh_ctrlblk.rcucblist,
54    .curtail = &rcu_bh_ctrlblk.rcucblist,
55};
56
57#ifdef CONFIG_DEBUG_LOCK_ALLOC
58int rcu_scheduler_active __read_mostly;
59EXPORT_SYMBOL_GPL(rcu_scheduler_active);
60#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
61
62#ifdef CONFIG_NO_HZ
63
64static long rcu_dynticks_nesting = 1;
65
66/*
67 * Enter dynticks-idle mode, which is an extended quiescent state
68 * if we have fully entered that mode (i.e., if the new value of
69 * dynticks_nesting is zero).
70 */
71void rcu_enter_nohz(void)
72{
73    if (--rcu_dynticks_nesting == 0)
74        rcu_sched_qs(0); /* implies rcu_bh_qsctr_inc(0) */
75}
76
77/*
78 * Exit dynticks-idle mode, so that we are no longer in an extended
79 * quiescent state.
80 */
81void rcu_exit_nohz(void)
82{
83    rcu_dynticks_nesting++;
84}
85
86#endif /* #ifdef CONFIG_NO_HZ */
87
88/*
89 * Helper function for rcu_qsctr_inc() and rcu_bh_qsctr_inc().
90 * Also disable irqs to avoid confusion due to interrupt handlers
91 * invoking call_rcu().
92 */
93static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
94{
95    unsigned long flags;
96
97    local_irq_save(flags);
98    if (rcp->rcucblist != NULL &&
99        rcp->donetail != rcp->curtail) {
100        rcp->donetail = rcp->curtail;
101        local_irq_restore(flags);
102        return 1;
103    }
104    local_irq_restore(flags);
105
106    return 0;
107}
108
109/*
110 * Record an rcu quiescent state. And an rcu_bh quiescent state while we
111 * are at it, given that any rcu quiescent state is also an rcu_bh
112 * quiescent state. Use "+" instead of "||" to defeat short circuiting.
113 */
114void rcu_sched_qs(int cpu)
115{
116    if (rcu_qsctr_help(&rcu_sched_ctrlblk) +
117        rcu_qsctr_help(&rcu_bh_ctrlblk))
118        raise_softirq(RCU_SOFTIRQ);
119}
120
121/*
122 * Record an rcu_bh quiescent state.
123 */
124void rcu_bh_qs(int cpu)
125{
126    if (rcu_qsctr_help(&rcu_bh_ctrlblk))
127        raise_softirq(RCU_SOFTIRQ);
128}
129
130/*
131 * Check to see if the scheduling-clock interrupt came from an extended
132 * quiescent state, and, if so, tell RCU about it.
133 */
134void rcu_check_callbacks(int cpu, int user)
135{
136    if (user ||
137        (idle_cpu(cpu) &&
138         !in_softirq() &&
139         hardirq_count() <= (1 << HARDIRQ_SHIFT)))
140        rcu_sched_qs(cpu);
141    else if (!in_softirq())
142        rcu_bh_qs(cpu);
143}
144
145/*
146 * Helper function for rcu_process_callbacks() that operates on the
147 * specified rcu_ctrlkblk structure.
148 */
149static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
150{
151    struct rcu_head *next, *list;
152    unsigned long flags;
153
154    /* If no RCU callbacks ready to invoke, just return. */
155    if (&rcp->rcucblist == rcp->donetail)
156        return;
157
158    /* Move the ready-to-invoke callbacks to a local list. */
159    local_irq_save(flags);
160    list = rcp->rcucblist;
161    rcp->rcucblist = *rcp->donetail;
162    *rcp->donetail = NULL;
163    if (rcp->curtail == rcp->donetail)
164        rcp->curtail = &rcp->rcucblist;
165    rcp->donetail = &rcp->rcucblist;
166    local_irq_restore(flags);
167
168    /* Invoke the callbacks on the local list. */
169    while (list) {
170        next = list->next;
171        prefetch(next);
172        debug_rcu_head_unqueue(list);
173        list->func(list);
174        list = next;
175    }
176}
177
178/*
179 * Invoke any callbacks whose grace period has completed.
180 */
181static void rcu_process_callbacks(struct softirq_action *unused)
182{
183    __rcu_process_callbacks(&rcu_sched_ctrlblk);
184    __rcu_process_callbacks(&rcu_bh_ctrlblk);
185}
186
187/*
188 * Wait for a grace period to elapse. But it is illegal to invoke
189 * synchronize_sched() from within an RCU read-side critical section.
190 * Therefore, any legal call to synchronize_sched() is a quiescent
191 * state, and so on a UP system, synchronize_sched() need do nothing.
192 * Ditto for synchronize_rcu_bh(). (But Lai Jiangshan points out the
193 * benefits of doing might_sleep() to reduce latency.)
194 *
195 * Cool, huh? (Due to Josh Triplett.)
196 *
197 * But we want to make this a static inline later. The cond_resched()
198 * currently makes this problematic.
199 */
200void synchronize_sched(void)
201{
202    cond_resched();
203}
204EXPORT_SYMBOL_GPL(synchronize_sched);
205
206/*
207 * Helper function for call_rcu() and call_rcu_bh().
208 */
209static void __call_rcu(struct rcu_head *head,
210               void (*func)(struct rcu_head *rcu),
211               struct rcu_ctrlblk *rcp)
212{
213    unsigned long flags;
214
215    debug_rcu_head_queue(head);
216    head->func = func;
217    head->next = NULL;
218
219    local_irq_save(flags);
220    *rcp->curtail = head;
221    rcp->curtail = &head->next;
222    local_irq_restore(flags);
223}
224
225/*
226 * Post an RCU callback to be invoked after the end of an RCU grace
227 * period. But since we have but one CPU, that would be after any
228 * quiescent state.
229 */
230void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
231{
232    __call_rcu(head, func, &rcu_sched_ctrlblk);
233}
234EXPORT_SYMBOL_GPL(call_rcu);
235
236/*
237 * Post an RCU bottom-half callback to be invoked after any subsequent
238 * quiescent state.
239 */
240void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
241{
242    __call_rcu(head, func, &rcu_bh_ctrlblk);
243}
244EXPORT_SYMBOL_GPL(call_rcu_bh);
245
246void rcu_barrier(void)
247{
248    struct rcu_synchronize rcu;
249
250    init_rcu_head_on_stack(&rcu.head);
251    init_completion(&rcu.completion);
252    /* Will wake me after RCU finished. */
253    call_rcu(&rcu.head, wakeme_after_rcu);
254    /* Wait for it. */
255    wait_for_completion(&rcu.completion);
256    destroy_rcu_head_on_stack(&rcu.head);
257}
258EXPORT_SYMBOL_GPL(rcu_barrier);
259
260void rcu_barrier_bh(void)
261{
262    struct rcu_synchronize rcu;
263
264    init_rcu_head_on_stack(&rcu.head);
265    init_completion(&rcu.completion);
266    /* Will wake me after RCU finished. */
267    call_rcu_bh(&rcu.head, wakeme_after_rcu);
268    /* Wait for it. */
269    wait_for_completion(&rcu.completion);
270    destroy_rcu_head_on_stack(&rcu.head);
271}
272EXPORT_SYMBOL_GPL(rcu_barrier_bh);
273
274void rcu_barrier_sched(void)
275{
276    struct rcu_synchronize rcu;
277
278    init_rcu_head_on_stack(&rcu.head);
279    init_completion(&rcu.completion);
280    /* Will wake me after RCU finished. */
281    call_rcu_sched(&rcu.head, wakeme_after_rcu);
282    /* Wait for it. */
283    wait_for_completion(&rcu.completion);
284    destroy_rcu_head_on_stack(&rcu.head);
285}
286EXPORT_SYMBOL_GPL(rcu_barrier_sched);
287
288void __init rcu_init(void)
289{
290    open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
291}
292
293#include "rcutiny_plugin.h"
294

Archive Download this file



interactive