Root/
1 | /* |
2 | * RT-Mutexes: simple blocking mutual exclusion locks with PI support |
3 | * |
4 | * started by Ingo Molnar and Thomas Gleixner. |
5 | * |
6 | * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
7 | * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com> |
8 | * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt |
9 | * Copyright (C) 2006 Esben Nielsen |
10 | * |
11 | * See Documentation/rt-mutex-design.txt for details. |
12 | */ |
13 | #include <linux/spinlock.h> |
14 | #include <linux/module.h> |
15 | #include <linux/sched.h> |
16 | #include <linux/timer.h> |
17 | |
18 | #include "rtmutex_common.h" |
19 | |
20 | /* |
21 | * lock->owner state tracking: |
22 | * |
23 | * lock->owner holds the task_struct pointer of the owner. Bit 0 and 1 |
24 | * are used to keep track of the "owner is pending" and "lock has |
25 | * waiters" state. |
26 | * |
27 | * owner bit1 bit0 |
28 | * NULL 0 0 lock is free (fast acquire possible) |
29 | * NULL 0 1 invalid state |
30 | * NULL 1 0 Transitional State* |
31 | * NULL 1 1 invalid state |
32 | * taskpointer 0 0 lock is held (fast release possible) |
33 | * taskpointer 0 1 task is pending owner |
34 | * taskpointer 1 0 lock is held and has waiters |
35 | * taskpointer 1 1 task is pending owner and lock has more waiters |
36 | * |
37 | * Pending ownership is assigned to the top (highest priority) |
38 | * waiter of the lock, when the lock is released. The thread is woken |
39 | * up and can now take the lock. Until the lock is taken (bit 0 |
40 | * cleared) a competing higher priority thread can steal the lock |
41 | * which puts the woken up thread back on the waiters list. |
42 | * |
43 | * The fast atomic compare exchange based acquire and release is only |
44 | * possible when bit 0 and 1 of lock->owner are 0. |
45 | * |
46 | * (*) There's a small time where the owner can be NULL and the |
47 | * "lock has waiters" bit is set. This can happen when grabbing the lock. |
48 | * To prevent a cmpxchg of the owner releasing the lock, we need to set this |
49 | * bit before looking at the lock, hence the reason this is a transitional |
50 | * state. |
51 | */ |
52 | |
53 | static void |
54 | rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner, |
55 | unsigned long mask) |
56 | { |
57 | unsigned long val = (unsigned long)owner | mask; |
58 | |
59 | if (rt_mutex_has_waiters(lock)) |
60 | val |= RT_MUTEX_HAS_WAITERS; |
61 | |
62 | lock->owner = (struct task_struct *)val; |
63 | } |
64 | |
65 | static inline void clear_rt_mutex_waiters(struct rt_mutex *lock) |
66 | { |
67 | lock->owner = (struct task_struct *) |
68 | ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); |
69 | } |
70 | |
71 | static void fixup_rt_mutex_waiters(struct rt_mutex *lock) |
72 | { |
73 | if (!rt_mutex_has_waiters(lock)) |
74 | clear_rt_mutex_waiters(lock); |
75 | } |
76 | |
77 | /* |
78 | * We can speed up the acquire/release, if the architecture |
79 | * supports cmpxchg and if there's no debugging state to be set up |
80 | */ |
81 | #if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES) |
82 | # define rt_mutex_cmpxchg(l,c,n) (cmpxchg(&l->owner, c, n) == c) |
83 | static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) |
84 | { |
85 | unsigned long owner, *p = (unsigned long *) &lock->owner; |
86 | |
87 | do { |
88 | owner = *p; |
89 | } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner); |
90 | } |
91 | #else |
92 | # define rt_mutex_cmpxchg(l,c,n) (0) |
93 | static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) |
94 | { |
95 | lock->owner = (struct task_struct *) |
96 | ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); |
97 | } |
98 | #endif |
99 | |
100 | /* |
101 | * Calculate task priority from the waiter list priority |
102 | * |
103 | * Return task->normal_prio when the waiter list is empty or when |
104 | * the waiter is not allowed to do priority boosting |
105 | */ |
106 | int rt_mutex_getprio(struct task_struct *task) |
107 | { |
108 | if (likely(!task_has_pi_waiters(task))) |
109 | return task->normal_prio; |
110 | |
111 | return min(task_top_pi_waiter(task)->pi_list_entry.prio, |
112 | task->normal_prio); |
113 | } |
114 | |
115 | /* |
116 | * Adjust the priority of a task, after its pi_waiters got modified. |
117 | * |
118 | * This can be both boosting and unboosting. task->pi_lock must be held. |
119 | */ |
120 | static void __rt_mutex_adjust_prio(struct task_struct *task) |
121 | { |
122 | int prio = rt_mutex_getprio(task); |
123 | |
124 | if (task->prio != prio) |
125 | rt_mutex_setprio(task, prio); |
126 | } |
127 | |
128 | /* |
129 | * Adjust task priority (undo boosting). Called from the exit path of |
130 | * rt_mutex_slowunlock() and rt_mutex_slowlock(). |
131 | * |
132 | * (Note: We do this outside of the protection of lock->wait_lock to |
133 | * allow the lock to be taken while or before we readjust the priority |
134 | * of task. We do not use the spin_xx_mutex() variants here as we are |
135 | * outside of the debug path.) |
136 | */ |
137 | static void rt_mutex_adjust_prio(struct task_struct *task) |
138 | { |
139 | unsigned long flags; |
140 | |
141 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
142 | __rt_mutex_adjust_prio(task); |
143 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
144 | } |
145 | |
146 | /* |
147 | * Max number of times we'll walk the boosting chain: |
148 | */ |
149 | int max_lock_depth = 1024; |
150 | |
151 | /* |
152 | * Adjust the priority chain. Also used for deadlock detection. |
153 | * Decreases task's usage by one - may thus free the task. |
154 | * Returns 0 or -EDEADLK. |
155 | */ |
156 | static int rt_mutex_adjust_prio_chain(struct task_struct *task, |
157 | int deadlock_detect, |
158 | struct rt_mutex *orig_lock, |
159 | struct rt_mutex_waiter *orig_waiter, |
160 | struct task_struct *top_task) |
161 | { |
162 | struct rt_mutex *lock; |
163 | struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter; |
164 | int detect_deadlock, ret = 0, depth = 0; |
165 | unsigned long flags; |
166 | |
167 | detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter, |
168 | deadlock_detect); |
169 | |
170 | /* |
171 | * The (de)boosting is a step by step approach with a lot of |
172 | * pitfalls. We want this to be preemptible and we want hold a |
173 | * maximum of two locks per step. So we have to check |
174 | * carefully whether things change under us. |
175 | */ |
176 | again: |
177 | if (++depth > max_lock_depth) { |
178 | static int prev_max; |
179 | |
180 | /* |
181 | * Print this only once. If the admin changes the limit, |
182 | * print a new message when reaching the limit again. |
183 | */ |
184 | if (prev_max != max_lock_depth) { |
185 | prev_max = max_lock_depth; |
186 | printk(KERN_WARNING "Maximum lock depth %d reached " |
187 | "task: %s (%d)\n", max_lock_depth, |
188 | top_task->comm, task_pid_nr(top_task)); |
189 | } |
190 | put_task_struct(task); |
191 | |
192 | return deadlock_detect ? -EDEADLK : 0; |
193 | } |
194 | retry: |
195 | /* |
196 | * Task can not go away as we did a get_task() before ! |
197 | */ |
198 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
199 | |
200 | waiter = task->pi_blocked_on; |
201 | /* |
202 | * Check whether the end of the boosting chain has been |
203 | * reached or the state of the chain has changed while we |
204 | * dropped the locks. |
205 | */ |
206 | if (!waiter || !waiter->task) |
207 | goto out_unlock_pi; |
208 | |
209 | /* |
210 | * Check the orig_waiter state. After we dropped the locks, |
211 | * the previous owner of the lock might have released the lock |
212 | * and made us the pending owner: |
213 | */ |
214 | if (orig_waiter && !orig_waiter->task) |
215 | goto out_unlock_pi; |
216 | |
217 | /* |
218 | * Drop out, when the task has no waiters. Note, |
219 | * top_waiter can be NULL, when we are in the deboosting |
220 | * mode! |
221 | */ |
222 | if (top_waiter && (!task_has_pi_waiters(task) || |
223 | top_waiter != task_top_pi_waiter(task))) |
224 | goto out_unlock_pi; |
225 | |
226 | /* |
227 | * When deadlock detection is off then we check, if further |
228 | * priority adjustment is necessary. |
229 | */ |
230 | if (!detect_deadlock && waiter->list_entry.prio == task->prio) |
231 | goto out_unlock_pi; |
232 | |
233 | lock = waiter->lock; |
234 | if (!raw_spin_trylock(&lock->wait_lock)) { |
235 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
236 | cpu_relax(); |
237 | goto retry; |
238 | } |
239 | |
240 | /* Deadlock detection */ |
241 | if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { |
242 | debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); |
243 | raw_spin_unlock(&lock->wait_lock); |
244 | ret = deadlock_detect ? -EDEADLK : 0; |
245 | goto out_unlock_pi; |
246 | } |
247 | |
248 | top_waiter = rt_mutex_top_waiter(lock); |
249 | |
250 | /* Requeue the waiter */ |
251 | plist_del(&waiter->list_entry, &lock->wait_list); |
252 | waiter->list_entry.prio = task->prio; |
253 | plist_add(&waiter->list_entry, &lock->wait_list); |
254 | |
255 | /* Release the task */ |
256 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
257 | put_task_struct(task); |
258 | |
259 | /* Grab the next task */ |
260 | task = rt_mutex_owner(lock); |
261 | get_task_struct(task); |
262 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
263 | |
264 | if (waiter == rt_mutex_top_waiter(lock)) { |
265 | /* Boost the owner */ |
266 | plist_del(&top_waiter->pi_list_entry, &task->pi_waiters); |
267 | waiter->pi_list_entry.prio = waiter->list_entry.prio; |
268 | plist_add(&waiter->pi_list_entry, &task->pi_waiters); |
269 | __rt_mutex_adjust_prio(task); |
270 | |
271 | } else if (top_waiter == waiter) { |
272 | /* Deboost the owner */ |
273 | plist_del(&waiter->pi_list_entry, &task->pi_waiters); |
274 | waiter = rt_mutex_top_waiter(lock); |
275 | waiter->pi_list_entry.prio = waiter->list_entry.prio; |
276 | plist_add(&waiter->pi_list_entry, &task->pi_waiters); |
277 | __rt_mutex_adjust_prio(task); |
278 | } |
279 | |
280 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
281 | |
282 | top_waiter = rt_mutex_top_waiter(lock); |
283 | raw_spin_unlock(&lock->wait_lock); |
284 | |
285 | if (!detect_deadlock && waiter != top_waiter) |
286 | goto out_put_task; |
287 | |
288 | goto again; |
289 | |
290 | out_unlock_pi: |
291 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
292 | out_put_task: |
293 | put_task_struct(task); |
294 | |
295 | return ret; |
296 | } |
297 | |
298 | /* |
299 | * Optimization: check if we can steal the lock from the |
300 | * assigned pending owner [which might not have taken the |
301 | * lock yet]: |
302 | */ |
303 | static inline int try_to_steal_lock(struct rt_mutex *lock, |
304 | struct task_struct *task) |
305 | { |
306 | struct task_struct *pendowner = rt_mutex_owner(lock); |
307 | struct rt_mutex_waiter *next; |
308 | unsigned long flags; |
309 | |
310 | if (!rt_mutex_owner_pending(lock)) |
311 | return 0; |
312 | |
313 | if (pendowner == task) |
314 | return 1; |
315 | |
316 | raw_spin_lock_irqsave(&pendowner->pi_lock, flags); |
317 | if (task->prio >= pendowner->prio) { |
318 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
319 | return 0; |
320 | } |
321 | |
322 | /* |
323 | * Check if a waiter is enqueued on the pending owners |
324 | * pi_waiters list. Remove it and readjust pending owners |
325 | * priority. |
326 | */ |
327 | if (likely(!rt_mutex_has_waiters(lock))) { |
328 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
329 | return 1; |
330 | } |
331 | |
332 | /* No chain handling, pending owner is not blocked on anything: */ |
333 | next = rt_mutex_top_waiter(lock); |
334 | plist_del(&next->pi_list_entry, &pendowner->pi_waiters); |
335 | __rt_mutex_adjust_prio(pendowner); |
336 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
337 | |
338 | /* |
339 | * We are going to steal the lock and a waiter was |
340 | * enqueued on the pending owners pi_waiters queue. So |
341 | * we have to enqueue this waiter into |
342 | * task->pi_waiters list. This covers the case, |
343 | * where task is boosted because it holds another |
344 | * lock and gets unboosted because the booster is |
345 | * interrupted, so we would delay a waiter with higher |
346 | * priority as task->normal_prio. |
347 | * |
348 | * Note: in the rare case of a SCHED_OTHER task changing |
349 | * its priority and thus stealing the lock, next->task |
350 | * might be task: |
351 | */ |
352 | if (likely(next->task != task)) { |
353 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
354 | plist_add(&next->pi_list_entry, &task->pi_waiters); |
355 | __rt_mutex_adjust_prio(task); |
356 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
357 | } |
358 | return 1; |
359 | } |
360 | |
361 | /* |
362 | * Try to take an rt-mutex |
363 | * |
364 | * This fails |
365 | * - when the lock has a real owner |
366 | * - when a different pending owner exists and has higher priority than current |
367 | * |
368 | * Must be called with lock->wait_lock held. |
369 | */ |
370 | static int try_to_take_rt_mutex(struct rt_mutex *lock) |
371 | { |
372 | /* |
373 | * We have to be careful here if the atomic speedups are |
374 | * enabled, such that, when |
375 | * - no other waiter is on the lock |
376 | * - the lock has been released since we did the cmpxchg |
377 | * the lock can be released or taken while we are doing the |
378 | * checks and marking the lock with RT_MUTEX_HAS_WAITERS. |
379 | * |
380 | * The atomic acquire/release aware variant of |
381 | * mark_rt_mutex_waiters uses a cmpxchg loop. After setting |
382 | * the WAITERS bit, the atomic release / acquire can not |
383 | * happen anymore and lock->wait_lock protects us from the |
384 | * non-atomic case. |
385 | * |
386 | * Note, that this might set lock->owner = |
387 | * RT_MUTEX_HAS_WAITERS in the case the lock is not contended |
388 | * any more. This is fixed up when we take the ownership. |
389 | * This is the transitional state explained at the top of this file. |
390 | */ |
391 | mark_rt_mutex_waiters(lock); |
392 | |
393 | if (rt_mutex_owner(lock) && !try_to_steal_lock(lock, current)) |
394 | return 0; |
395 | |
396 | /* We got the lock. */ |
397 | debug_rt_mutex_lock(lock); |
398 | |
399 | rt_mutex_set_owner(lock, current, 0); |
400 | |
401 | rt_mutex_deadlock_account_lock(lock, current); |
402 | |
403 | return 1; |
404 | } |
405 | |
406 | /* |
407 | * Task blocks on lock. |
408 | * |
409 | * Prepare waiter and propagate pi chain |
410 | * |
411 | * This must be called with lock->wait_lock held. |
412 | */ |
413 | static int task_blocks_on_rt_mutex(struct rt_mutex *lock, |
414 | struct rt_mutex_waiter *waiter, |
415 | struct task_struct *task, |
416 | int detect_deadlock) |
417 | { |
418 | struct task_struct *owner = rt_mutex_owner(lock); |
419 | struct rt_mutex_waiter *top_waiter = waiter; |
420 | unsigned long flags; |
421 | int chain_walk = 0, res; |
422 | |
423 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
424 | __rt_mutex_adjust_prio(task); |
425 | waiter->task = task; |
426 | waiter->lock = lock; |
427 | plist_node_init(&waiter->list_entry, task->prio); |
428 | plist_node_init(&waiter->pi_list_entry, task->prio); |
429 | |
430 | /* Get the top priority waiter on the lock */ |
431 | if (rt_mutex_has_waiters(lock)) |
432 | top_waiter = rt_mutex_top_waiter(lock); |
433 | plist_add(&waiter->list_entry, &lock->wait_list); |
434 | |
435 | task->pi_blocked_on = waiter; |
436 | |
437 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
438 | |
439 | if (waiter == rt_mutex_top_waiter(lock)) { |
440 | raw_spin_lock_irqsave(&owner->pi_lock, flags); |
441 | plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters); |
442 | plist_add(&waiter->pi_list_entry, &owner->pi_waiters); |
443 | |
444 | __rt_mutex_adjust_prio(owner); |
445 | if (owner->pi_blocked_on) |
446 | chain_walk = 1; |
447 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); |
448 | } |
449 | else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) |
450 | chain_walk = 1; |
451 | |
452 | if (!chain_walk) |
453 | return 0; |
454 | |
455 | /* |
456 | * The owner can't disappear while holding a lock, |
457 | * so the owner struct is protected by wait_lock. |
458 | * Gets dropped in rt_mutex_adjust_prio_chain()! |
459 | */ |
460 | get_task_struct(owner); |
461 | |
462 | raw_spin_unlock(&lock->wait_lock); |
463 | |
464 | res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter, |
465 | task); |
466 | |
467 | raw_spin_lock(&lock->wait_lock); |
468 | |
469 | return res; |
470 | } |
471 | |
472 | /* |
473 | * Wake up the next waiter on the lock. |
474 | * |
475 | * Remove the top waiter from the current tasks waiter list and from |
476 | * the lock waiter list. Set it as pending owner. Then wake it up. |
477 | * |
478 | * Called with lock->wait_lock held. |
479 | */ |
480 | static void wakeup_next_waiter(struct rt_mutex *lock) |
481 | { |
482 | struct rt_mutex_waiter *waiter; |
483 | struct task_struct *pendowner; |
484 | unsigned long flags; |
485 | |
486 | raw_spin_lock_irqsave(¤t->pi_lock, flags); |
487 | |
488 | waiter = rt_mutex_top_waiter(lock); |
489 | plist_del(&waiter->list_entry, &lock->wait_list); |
490 | |
491 | /* |
492 | * Remove it from current->pi_waiters. We do not adjust a |
493 | * possible priority boost right now. We execute wakeup in the |
494 | * boosted mode and go back to normal after releasing |
495 | * lock->wait_lock. |
496 | */ |
497 | plist_del(&waiter->pi_list_entry, ¤t->pi_waiters); |
498 | pendowner = waiter->task; |
499 | waiter->task = NULL; |
500 | |
501 | rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING); |
502 | |
503 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); |
504 | |
505 | /* |
506 | * Clear the pi_blocked_on variable and enqueue a possible |
507 | * waiter into the pi_waiters list of the pending owner. This |
508 | * prevents that in case the pending owner gets unboosted a |
509 | * waiter with higher priority than pending-owner->normal_prio |
510 | * is blocked on the unboosted (pending) owner. |
511 | */ |
512 | raw_spin_lock_irqsave(&pendowner->pi_lock, flags); |
513 | |
514 | WARN_ON(!pendowner->pi_blocked_on); |
515 | WARN_ON(pendowner->pi_blocked_on != waiter); |
516 | WARN_ON(pendowner->pi_blocked_on->lock != lock); |
517 | |
518 | pendowner->pi_blocked_on = NULL; |
519 | |
520 | if (rt_mutex_has_waiters(lock)) { |
521 | struct rt_mutex_waiter *next; |
522 | |
523 | next = rt_mutex_top_waiter(lock); |
524 | plist_add(&next->pi_list_entry, &pendowner->pi_waiters); |
525 | } |
526 | raw_spin_unlock_irqrestore(&pendowner->pi_lock, flags); |
527 | |
528 | wake_up_process(pendowner); |
529 | } |
530 | |
531 | /* |
532 | * Remove a waiter from a lock |
533 | * |
534 | * Must be called with lock->wait_lock held |
535 | */ |
536 | static void remove_waiter(struct rt_mutex *lock, |
537 | struct rt_mutex_waiter *waiter) |
538 | { |
539 | int first = (waiter == rt_mutex_top_waiter(lock)); |
540 | struct task_struct *owner = rt_mutex_owner(lock); |
541 | unsigned long flags; |
542 | int chain_walk = 0; |
543 | |
544 | raw_spin_lock_irqsave(¤t->pi_lock, flags); |
545 | plist_del(&waiter->list_entry, &lock->wait_list); |
546 | waiter->task = NULL; |
547 | current->pi_blocked_on = NULL; |
548 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); |
549 | |
550 | if (first && owner != current) { |
551 | |
552 | raw_spin_lock_irqsave(&owner->pi_lock, flags); |
553 | |
554 | plist_del(&waiter->pi_list_entry, &owner->pi_waiters); |
555 | |
556 | if (rt_mutex_has_waiters(lock)) { |
557 | struct rt_mutex_waiter *next; |
558 | |
559 | next = rt_mutex_top_waiter(lock); |
560 | plist_add(&next->pi_list_entry, &owner->pi_waiters); |
561 | } |
562 | __rt_mutex_adjust_prio(owner); |
563 | |
564 | if (owner->pi_blocked_on) |
565 | chain_walk = 1; |
566 | |
567 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); |
568 | } |
569 | |
570 | WARN_ON(!plist_node_empty(&waiter->pi_list_entry)); |
571 | |
572 | if (!chain_walk) |
573 | return; |
574 | |
575 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ |
576 | get_task_struct(owner); |
577 | |
578 | raw_spin_unlock(&lock->wait_lock); |
579 | |
580 | rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current); |
581 | |
582 | raw_spin_lock(&lock->wait_lock); |
583 | } |
584 | |
585 | /* |
586 | * Recheck the pi chain, in case we got a priority setting |
587 | * |
588 | * Called from sched_setscheduler |
589 | */ |
590 | void rt_mutex_adjust_pi(struct task_struct *task) |
591 | { |
592 | struct rt_mutex_waiter *waiter; |
593 | unsigned long flags; |
594 | |
595 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
596 | |
597 | waiter = task->pi_blocked_on; |
598 | if (!waiter || waiter->list_entry.prio == task->prio) { |
599 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
600 | return; |
601 | } |
602 | |
603 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
604 | |
605 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ |
606 | get_task_struct(task); |
607 | rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task); |
608 | } |
609 | |
610 | /** |
611 | * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop |
612 | * @lock: the rt_mutex to take |
613 | * @state: the state the task should block in (TASK_INTERRUPTIBLE |
614 | * or TASK_UNINTERRUPTIBLE) |
615 | * @timeout: the pre-initialized and started timer, or NULL for none |
616 | * @waiter: the pre-initialized rt_mutex_waiter |
617 | * @detect_deadlock: passed to task_blocks_on_rt_mutex |
618 | * |
619 | * lock->wait_lock must be held by the caller. |
620 | */ |
621 | static int __sched |
622 | __rt_mutex_slowlock(struct rt_mutex *lock, int state, |
623 | struct hrtimer_sleeper *timeout, |
624 | struct rt_mutex_waiter *waiter, |
625 | int detect_deadlock) |
626 | { |
627 | int ret = 0; |
628 | |
629 | for (;;) { |
630 | /* Try to acquire the lock: */ |
631 | if (try_to_take_rt_mutex(lock)) |
632 | break; |
633 | |
634 | /* |
635 | * TASK_INTERRUPTIBLE checks for signals and |
636 | * timeout. Ignored otherwise. |
637 | */ |
638 | if (unlikely(state == TASK_INTERRUPTIBLE)) { |
639 | /* Signal pending? */ |
640 | if (signal_pending(current)) |
641 | ret = -EINTR; |
642 | if (timeout && !timeout->task) |
643 | ret = -ETIMEDOUT; |
644 | if (ret) |
645 | break; |
646 | } |
647 | |
648 | /* |
649 | * waiter->task is NULL the first time we come here and |
650 | * when we have been woken up by the previous owner |
651 | * but the lock got stolen by a higher prio task. |
652 | */ |
653 | if (!waiter->task) { |
654 | ret = task_blocks_on_rt_mutex(lock, waiter, current, |
655 | detect_deadlock); |
656 | /* |
657 | * If we got woken up by the owner then start loop |
658 | * all over without going into schedule to try |
659 | * to get the lock now: |
660 | */ |
661 | if (unlikely(!waiter->task)) { |
662 | /* |
663 | * Reset the return value. We might |
664 | * have returned with -EDEADLK and the |
665 | * owner released the lock while we |
666 | * were walking the pi chain. |
667 | */ |
668 | ret = 0; |
669 | continue; |
670 | } |
671 | if (unlikely(ret)) |
672 | break; |
673 | } |
674 | |
675 | raw_spin_unlock(&lock->wait_lock); |
676 | |
677 | debug_rt_mutex_print_deadlock(waiter); |
678 | |
679 | if (waiter->task) |
680 | schedule_rt_mutex(lock); |
681 | |
682 | raw_spin_lock(&lock->wait_lock); |
683 | set_current_state(state); |
684 | } |
685 | |
686 | return ret; |
687 | } |
688 | |
689 | /* |
690 | * Slow path lock function: |
691 | */ |
692 | static int __sched |
693 | rt_mutex_slowlock(struct rt_mutex *lock, int state, |
694 | struct hrtimer_sleeper *timeout, |
695 | int detect_deadlock) |
696 | { |
697 | struct rt_mutex_waiter waiter; |
698 | int ret = 0; |
699 | |
700 | debug_rt_mutex_init_waiter(&waiter); |
701 | waiter.task = NULL; |
702 | |
703 | raw_spin_lock(&lock->wait_lock); |
704 | |
705 | /* Try to acquire the lock again: */ |
706 | if (try_to_take_rt_mutex(lock)) { |
707 | raw_spin_unlock(&lock->wait_lock); |
708 | return 0; |
709 | } |
710 | |
711 | set_current_state(state); |
712 | |
713 | /* Setup the timer, when timeout != NULL */ |
714 | if (unlikely(timeout)) { |
715 | hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS); |
716 | if (!hrtimer_active(&timeout->timer)) |
717 | timeout->task = NULL; |
718 | } |
719 | |
720 | ret = __rt_mutex_slowlock(lock, state, timeout, &waiter, |
721 | detect_deadlock); |
722 | |
723 | set_current_state(TASK_RUNNING); |
724 | |
725 | if (unlikely(waiter.task)) |
726 | remove_waiter(lock, &waiter); |
727 | |
728 | /* |
729 | * try_to_take_rt_mutex() sets the waiter bit |
730 | * unconditionally. We might have to fix that up. |
731 | */ |
732 | fixup_rt_mutex_waiters(lock); |
733 | |
734 | raw_spin_unlock(&lock->wait_lock); |
735 | |
736 | /* Remove pending timer: */ |
737 | if (unlikely(timeout)) |
738 | hrtimer_cancel(&timeout->timer); |
739 | |
740 | /* |
741 | * Readjust priority, when we did not get the lock. We might |
742 | * have been the pending owner and boosted. Since we did not |
743 | * take the lock, the PI boost has to go. |
744 | */ |
745 | if (unlikely(ret)) |
746 | rt_mutex_adjust_prio(current); |
747 | |
748 | debug_rt_mutex_free_waiter(&waiter); |
749 | |
750 | return ret; |
751 | } |
752 | |
753 | /* |
754 | * Slow path try-lock function: |
755 | */ |
756 | static inline int |
757 | rt_mutex_slowtrylock(struct rt_mutex *lock) |
758 | { |
759 | int ret = 0; |
760 | |
761 | raw_spin_lock(&lock->wait_lock); |
762 | |
763 | if (likely(rt_mutex_owner(lock) != current)) { |
764 | |
765 | ret = try_to_take_rt_mutex(lock); |
766 | /* |
767 | * try_to_take_rt_mutex() sets the lock waiters |
768 | * bit unconditionally. Clean this up. |
769 | */ |
770 | fixup_rt_mutex_waiters(lock); |
771 | } |
772 | |
773 | raw_spin_unlock(&lock->wait_lock); |
774 | |
775 | return ret; |
776 | } |
777 | |
778 | /* |
779 | * Slow path to release a rt-mutex: |
780 | */ |
781 | static void __sched |
782 | rt_mutex_slowunlock(struct rt_mutex *lock) |
783 | { |
784 | raw_spin_lock(&lock->wait_lock); |
785 | |
786 | debug_rt_mutex_unlock(lock); |
787 | |
788 | rt_mutex_deadlock_account_unlock(current); |
789 | |
790 | if (!rt_mutex_has_waiters(lock)) { |
791 | lock->owner = NULL; |
792 | raw_spin_unlock(&lock->wait_lock); |
793 | return; |
794 | } |
795 | |
796 | wakeup_next_waiter(lock); |
797 | |
798 | raw_spin_unlock(&lock->wait_lock); |
799 | |
800 | /* Undo pi boosting if necessary: */ |
801 | rt_mutex_adjust_prio(current); |
802 | } |
803 | |
804 | /* |
805 | * debug aware fast / slowpath lock,trylock,unlock |
806 | * |
807 | * The atomic acquire/release ops are compiled away, when either the |
808 | * architecture does not support cmpxchg or when debugging is enabled. |
809 | */ |
810 | static inline int |
811 | rt_mutex_fastlock(struct rt_mutex *lock, int state, |
812 | int detect_deadlock, |
813 | int (*slowfn)(struct rt_mutex *lock, int state, |
814 | struct hrtimer_sleeper *timeout, |
815 | int detect_deadlock)) |
816 | { |
817 | if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { |
818 | rt_mutex_deadlock_account_lock(lock, current); |
819 | return 0; |
820 | } else |
821 | return slowfn(lock, state, NULL, detect_deadlock); |
822 | } |
823 | |
824 | static inline int |
825 | rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, |
826 | struct hrtimer_sleeper *timeout, int detect_deadlock, |
827 | int (*slowfn)(struct rt_mutex *lock, int state, |
828 | struct hrtimer_sleeper *timeout, |
829 | int detect_deadlock)) |
830 | { |
831 | if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { |
832 | rt_mutex_deadlock_account_lock(lock, current); |
833 | return 0; |
834 | } else |
835 | return slowfn(lock, state, timeout, detect_deadlock); |
836 | } |
837 | |
838 | static inline int |
839 | rt_mutex_fasttrylock(struct rt_mutex *lock, |
840 | int (*slowfn)(struct rt_mutex *lock)) |
841 | { |
842 | if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { |
843 | rt_mutex_deadlock_account_lock(lock, current); |
844 | return 1; |
845 | } |
846 | return slowfn(lock); |
847 | } |
848 | |
849 | static inline void |
850 | rt_mutex_fastunlock(struct rt_mutex *lock, |
851 | void (*slowfn)(struct rt_mutex *lock)) |
852 | { |
853 | if (likely(rt_mutex_cmpxchg(lock, current, NULL))) |
854 | rt_mutex_deadlock_account_unlock(current); |
855 | else |
856 | slowfn(lock); |
857 | } |
858 | |
859 | /** |
860 | * rt_mutex_lock - lock a rt_mutex |
861 | * |
862 | * @lock: the rt_mutex to be locked |
863 | */ |
864 | void __sched rt_mutex_lock(struct rt_mutex *lock) |
865 | { |
866 | might_sleep(); |
867 | |
868 | rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock); |
869 | } |
870 | EXPORT_SYMBOL_GPL(rt_mutex_lock); |
871 | |
872 | /** |
873 | * rt_mutex_lock_interruptible - lock a rt_mutex interruptible |
874 | * |
875 | * @lock: the rt_mutex to be locked |
876 | * @detect_deadlock: deadlock detection on/off |
877 | * |
878 | * Returns: |
879 | * 0 on success |
880 | * -EINTR when interrupted by a signal |
881 | * -EDEADLK when the lock would deadlock (when deadlock detection is on) |
882 | */ |
883 | int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock, |
884 | int detect_deadlock) |
885 | { |
886 | might_sleep(); |
887 | |
888 | return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, |
889 | detect_deadlock, rt_mutex_slowlock); |
890 | } |
891 | EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); |
892 | |
893 | /** |
894 | * rt_mutex_timed_lock - lock a rt_mutex interruptible |
895 | * the timeout structure is provided |
896 | * by the caller |
897 | * |
898 | * @lock: the rt_mutex to be locked |
899 | * @timeout: timeout structure or NULL (no timeout) |
900 | * @detect_deadlock: deadlock detection on/off |
901 | * |
902 | * Returns: |
903 | * 0 on success |
904 | * -EINTR when interrupted by a signal |
905 | * -ETIMEDOUT when the timeout expired |
906 | * -EDEADLK when the lock would deadlock (when deadlock detection is on) |
907 | */ |
908 | int |
909 | rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout, |
910 | int detect_deadlock) |
911 | { |
912 | might_sleep(); |
913 | |
914 | return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, |
915 | detect_deadlock, rt_mutex_slowlock); |
916 | } |
917 | EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); |
918 | |
919 | /** |
920 | * rt_mutex_trylock - try to lock a rt_mutex |
921 | * |
922 | * @lock: the rt_mutex to be locked |
923 | * |
924 | * Returns 1 on success and 0 on contention |
925 | */ |
926 | int __sched rt_mutex_trylock(struct rt_mutex *lock) |
927 | { |
928 | return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock); |
929 | } |
930 | EXPORT_SYMBOL_GPL(rt_mutex_trylock); |
931 | |
932 | /** |
933 | * rt_mutex_unlock - unlock a rt_mutex |
934 | * |
935 | * @lock: the rt_mutex to be unlocked |
936 | */ |
937 | void __sched rt_mutex_unlock(struct rt_mutex *lock) |
938 | { |
939 | rt_mutex_fastunlock(lock, rt_mutex_slowunlock); |
940 | } |
941 | EXPORT_SYMBOL_GPL(rt_mutex_unlock); |
942 | |
943 | /** |
944 | * rt_mutex_destroy - mark a mutex unusable |
945 | * @lock: the mutex to be destroyed |
946 | * |
947 | * This function marks the mutex uninitialized, and any subsequent |
948 | * use of the mutex is forbidden. The mutex must not be locked when |
949 | * this function is called. |
950 | */ |
951 | void rt_mutex_destroy(struct rt_mutex *lock) |
952 | { |
953 | WARN_ON(rt_mutex_is_locked(lock)); |
954 | #ifdef CONFIG_DEBUG_RT_MUTEXES |
955 | lock->magic = NULL; |
956 | #endif |
957 | } |
958 | |
959 | EXPORT_SYMBOL_GPL(rt_mutex_destroy); |
960 | |
961 | /** |
962 | * __rt_mutex_init - initialize the rt lock |
963 | * |
964 | * @lock: the rt lock to be initialized |
965 | * |
966 | * Initialize the rt lock to unlocked state. |
967 | * |
968 | * Initializing of a locked rt lock is not allowed |
969 | */ |
970 | void __rt_mutex_init(struct rt_mutex *lock, const char *name) |
971 | { |
972 | lock->owner = NULL; |
973 | raw_spin_lock_init(&lock->wait_lock); |
974 | plist_head_init_raw(&lock->wait_list, &lock->wait_lock); |
975 | |
976 | debug_rt_mutex_init(lock, name); |
977 | } |
978 | EXPORT_SYMBOL_GPL(__rt_mutex_init); |
979 | |
980 | /** |
981 | * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a |
982 | * proxy owner |
983 | * |
984 | * @lock: the rt_mutex to be locked |
985 | * @proxy_owner:the task to set as owner |
986 | * |
987 | * No locking. Caller has to do serializing itself |
988 | * Special API call for PI-futex support |
989 | */ |
990 | void rt_mutex_init_proxy_locked(struct rt_mutex *lock, |
991 | struct task_struct *proxy_owner) |
992 | { |
993 | __rt_mutex_init(lock, NULL); |
994 | debug_rt_mutex_proxy_lock(lock, proxy_owner); |
995 | rt_mutex_set_owner(lock, proxy_owner, 0); |
996 | rt_mutex_deadlock_account_lock(lock, proxy_owner); |
997 | } |
998 | |
999 | /** |
1000 | * rt_mutex_proxy_unlock - release a lock on behalf of owner |
1001 | * |
1002 | * @lock: the rt_mutex to be locked |
1003 | * |
1004 | * No locking. Caller has to do serializing itself |
1005 | * Special API call for PI-futex support |
1006 | */ |
1007 | void rt_mutex_proxy_unlock(struct rt_mutex *lock, |
1008 | struct task_struct *proxy_owner) |
1009 | { |
1010 | debug_rt_mutex_proxy_unlock(lock); |
1011 | rt_mutex_set_owner(lock, NULL, 0); |
1012 | rt_mutex_deadlock_account_unlock(proxy_owner); |
1013 | } |
1014 | |
1015 | /** |
1016 | * rt_mutex_start_proxy_lock() - Start lock acquisition for another task |
1017 | * @lock: the rt_mutex to take |
1018 | * @waiter: the pre-initialized rt_mutex_waiter |
1019 | * @task: the task to prepare |
1020 | * @detect_deadlock: perform deadlock detection (1) or not (0) |
1021 | * |
1022 | * Returns: |
1023 | * 0 - task blocked on lock |
1024 | * 1 - acquired the lock for task, caller should wake it up |
1025 | * <0 - error |
1026 | * |
1027 | * Special API call for FUTEX_REQUEUE_PI support. |
1028 | */ |
1029 | int rt_mutex_start_proxy_lock(struct rt_mutex *lock, |
1030 | struct rt_mutex_waiter *waiter, |
1031 | struct task_struct *task, int detect_deadlock) |
1032 | { |
1033 | int ret; |
1034 | |
1035 | raw_spin_lock(&lock->wait_lock); |
1036 | |
1037 | mark_rt_mutex_waiters(lock); |
1038 | |
1039 | if (!rt_mutex_owner(lock) || try_to_steal_lock(lock, task)) { |
1040 | /* We got the lock for task. */ |
1041 | debug_rt_mutex_lock(lock); |
1042 | rt_mutex_set_owner(lock, task, 0); |
1043 | raw_spin_unlock(&lock->wait_lock); |
1044 | rt_mutex_deadlock_account_lock(lock, task); |
1045 | return 1; |
1046 | } |
1047 | |
1048 | ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock); |
1049 | |
1050 | if (ret && !waiter->task) { |
1051 | /* |
1052 | * Reset the return value. We might have |
1053 | * returned with -EDEADLK and the owner |
1054 | * released the lock while we were walking the |
1055 | * pi chain. Let the waiter sort it out. |
1056 | */ |
1057 | ret = 0; |
1058 | } |
1059 | raw_spin_unlock(&lock->wait_lock); |
1060 | |
1061 | debug_rt_mutex_print_deadlock(waiter); |
1062 | |
1063 | return ret; |
1064 | } |
1065 | |
1066 | /** |
1067 | * rt_mutex_next_owner - return the next owner of the lock |
1068 | * |
1069 | * @lock: the rt lock query |
1070 | * |
1071 | * Returns the next owner of the lock or NULL |
1072 | * |
1073 | * Caller has to serialize against other accessors to the lock |
1074 | * itself. |
1075 | * |
1076 | * Special API call for PI-futex support |
1077 | */ |
1078 | struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock) |
1079 | { |
1080 | if (!rt_mutex_has_waiters(lock)) |
1081 | return NULL; |
1082 | |
1083 | return rt_mutex_top_waiter(lock)->task; |
1084 | } |
1085 | |
1086 | /** |
1087 | * rt_mutex_finish_proxy_lock() - Complete lock acquisition |
1088 | * @lock: the rt_mutex we were woken on |
1089 | * @to: the timeout, null if none. hrtimer should already have |
1090 | * been started. |
1091 | * @waiter: the pre-initialized rt_mutex_waiter |
1092 | * @detect_deadlock: perform deadlock detection (1) or not (0) |
1093 | * |
1094 | * Complete the lock acquisition started our behalf by another thread. |
1095 | * |
1096 | * Returns: |
1097 | * 0 - success |
1098 | * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK |
1099 | * |
1100 | * Special API call for PI-futex requeue support |
1101 | */ |
1102 | int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, |
1103 | struct hrtimer_sleeper *to, |
1104 | struct rt_mutex_waiter *waiter, |
1105 | int detect_deadlock) |
1106 | { |
1107 | int ret; |
1108 | |
1109 | raw_spin_lock(&lock->wait_lock); |
1110 | |
1111 | set_current_state(TASK_INTERRUPTIBLE); |
1112 | |
1113 | ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, |
1114 | detect_deadlock); |
1115 | |
1116 | set_current_state(TASK_RUNNING); |
1117 | |
1118 | if (unlikely(waiter->task)) |
1119 | remove_waiter(lock, waiter); |
1120 | |
1121 | /* |
1122 | * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might |
1123 | * have to fix that up. |
1124 | */ |
1125 | fixup_rt_mutex_waiters(lock); |
1126 | |
1127 | raw_spin_unlock(&lock->wait_lock); |
1128 | |
1129 | /* |
1130 | * Readjust priority, when we did not get the lock. We might have been |
1131 | * the pending owner and boosted. Since we did not take the lock, the |
1132 | * PI boost has to go. |
1133 | */ |
1134 | if (unlikely(ret)) |
1135 | rt_mutex_adjust_prio(current); |
1136 | |
1137 | return ret; |
1138 | } |
1139 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9