Root/kernel/rtmutex.c

1/*
2 * RT-Mutexes: simple blocking mutual exclusion locks with PI support
3 *
4 * started by Ingo Molnar and Thomas Gleixner.
5 *
6 * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7 * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
8 * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
9 * Copyright (C) 2006 Esben Nielsen
10 *
11 * See Documentation/rt-mutex-design.txt for details.
12 */
13#include <linux/spinlock.h>
14#include <linux/export.h>
15#include <linux/sched.h>
16#include <linux/timer.h>
17
18#include "rtmutex_common.h"
19
20/*
21 * lock->owner state tracking:
22 *
23 * lock->owner holds the task_struct pointer of the owner. Bit 0
24 * is used to keep track of the "lock has waiters" state.
25 *
26 * owner bit0
27 * NULL 0 lock is free (fast acquire possible)
28 * NULL 1 lock is free and has waiters and the top waiter
29 * is going to take the lock*
30 * taskpointer 0 lock is held (fast release possible)
31 * taskpointer 1 lock is held and has waiters**
32 *
33 * The fast atomic compare exchange based acquire and release is only
34 * possible when bit 0 of lock->owner is 0.
35 *
36 * (*) It also can be a transitional state when grabbing the lock
37 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
38 * we need to set the bit0 before looking at the lock, and the owner may be
39 * NULL in this small time, hence this can be a transitional state.
40 *
41 * (**) There is a small time when bit 0 is set but there are no
42 * waiters. This can happen when grabbing the lock in the slow path.
43 * To prevent a cmpxchg of the owner releasing the lock, we need to
44 * set this bit before looking at the lock.
45 */
46
47static void
48rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
49{
50    unsigned long val = (unsigned long)owner;
51
52    if (rt_mutex_has_waiters(lock))
53        val |= RT_MUTEX_HAS_WAITERS;
54
55    lock->owner = (struct task_struct *)val;
56}
57
58static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
59{
60    lock->owner = (struct task_struct *)
61            ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
62}
63
64static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
65{
66    if (!rt_mutex_has_waiters(lock))
67        clear_rt_mutex_waiters(lock);
68}
69
70/*
71 * We can speed up the acquire/release, if the architecture
72 * supports cmpxchg and if there's no debugging state to be set up
73 */
74#if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
75# define rt_mutex_cmpxchg(l,c,n) (cmpxchg(&l->owner, c, n) == c)
76static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
77{
78    unsigned long owner, *p = (unsigned long *) &lock->owner;
79
80    do {
81        owner = *p;
82    } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
83}
84#else
85# define rt_mutex_cmpxchg(l,c,n) (0)
86static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
87{
88    lock->owner = (struct task_struct *)
89            ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
90}
91#endif
92
93/*
94 * Calculate task priority from the waiter list priority
95 *
96 * Return task->normal_prio when the waiter list is empty or when
97 * the waiter is not allowed to do priority boosting
98 */
99int rt_mutex_getprio(struct task_struct *task)
100{
101    if (likely(!task_has_pi_waiters(task)))
102        return task->normal_prio;
103
104    return min(task_top_pi_waiter(task)->pi_list_entry.prio,
105           task->normal_prio);
106}
107
108/*
109 * Adjust the priority of a task, after its pi_waiters got modified.
110 *
111 * This can be both boosting and unboosting. task->pi_lock must be held.
112 */
113static void __rt_mutex_adjust_prio(struct task_struct *task)
114{
115    int prio = rt_mutex_getprio(task);
116
117    if (task->prio != prio)
118        rt_mutex_setprio(task, prio);
119}
120
121/*
122 * Adjust task priority (undo boosting). Called from the exit path of
123 * rt_mutex_slowunlock() and rt_mutex_slowlock().
124 *
125 * (Note: We do this outside of the protection of lock->wait_lock to
126 * allow the lock to be taken while or before we readjust the priority
127 * of task. We do not use the spin_xx_mutex() variants here as we are
128 * outside of the debug path.)
129 */
130static void rt_mutex_adjust_prio(struct task_struct *task)
131{
132    unsigned long flags;
133
134    raw_spin_lock_irqsave(&task->pi_lock, flags);
135    __rt_mutex_adjust_prio(task);
136    raw_spin_unlock_irqrestore(&task->pi_lock, flags);
137}
138
139/*
140 * Max number of times we'll walk the boosting chain:
141 */
142int max_lock_depth = 1024;
143
144/*
145 * Adjust the priority chain. Also used for deadlock detection.
146 * Decreases task's usage by one - may thus free the task.
147 * Returns 0 or -EDEADLK.
148 */
149static int rt_mutex_adjust_prio_chain(struct task_struct *task,
150                      int deadlock_detect,
151                      struct rt_mutex *orig_lock,
152                      struct rt_mutex_waiter *orig_waiter,
153                      struct task_struct *top_task)
154{
155    struct rt_mutex *lock;
156    struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
157    int detect_deadlock, ret = 0, depth = 0;
158    unsigned long flags;
159
160    detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
161                             deadlock_detect);
162
163    /*
164     * The (de)boosting is a step by step approach with a lot of
165     * pitfalls. We want this to be preemptible and we want hold a
166     * maximum of two locks per step. So we have to check
167     * carefully whether things change under us.
168     */
169 again:
170    if (++depth > max_lock_depth) {
171        static int prev_max;
172
173        /*
174         * Print this only once. If the admin changes the limit,
175         * print a new message when reaching the limit again.
176         */
177        if (prev_max != max_lock_depth) {
178            prev_max = max_lock_depth;
179            printk(KERN_WARNING "Maximum lock depth %d reached "
180                   "task: %s (%d)\n", max_lock_depth,
181                   top_task->comm, task_pid_nr(top_task));
182        }
183        put_task_struct(task);
184
185        return deadlock_detect ? -EDEADLK : 0;
186    }
187 retry:
188    /*
189     * Task can not go away as we did a get_task() before !
190     */
191    raw_spin_lock_irqsave(&task->pi_lock, flags);
192
193    waiter = task->pi_blocked_on;
194    /*
195     * Check whether the end of the boosting chain has been
196     * reached or the state of the chain has changed while we
197     * dropped the locks.
198     */
199    if (!waiter)
200        goto out_unlock_pi;
201
202    /*
203     * Check the orig_waiter state. After we dropped the locks,
204     * the previous owner of the lock might have released the lock.
205     */
206    if (orig_waiter && !rt_mutex_owner(orig_lock))
207        goto out_unlock_pi;
208
209    /*
210     * Drop out, when the task has no waiters. Note,
211     * top_waiter can be NULL, when we are in the deboosting
212     * mode!
213     */
214    if (top_waiter && (!task_has_pi_waiters(task) ||
215               top_waiter != task_top_pi_waiter(task)))
216        goto out_unlock_pi;
217
218    /*
219     * When deadlock detection is off then we check, if further
220     * priority adjustment is necessary.
221     */
222    if (!detect_deadlock && waiter->list_entry.prio == task->prio)
223        goto out_unlock_pi;
224
225    lock = waiter->lock;
226    if (!raw_spin_trylock(&lock->wait_lock)) {
227        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
228        cpu_relax();
229        goto retry;
230    }
231
232    /* Deadlock detection */
233    if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
234        debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
235        raw_spin_unlock(&lock->wait_lock);
236        ret = deadlock_detect ? -EDEADLK : 0;
237        goto out_unlock_pi;
238    }
239
240    top_waiter = rt_mutex_top_waiter(lock);
241
242    /* Requeue the waiter */
243    plist_del(&waiter->list_entry, &lock->wait_list);
244    waiter->list_entry.prio = task->prio;
245    plist_add(&waiter->list_entry, &lock->wait_list);
246
247    /* Release the task */
248    raw_spin_unlock_irqrestore(&task->pi_lock, flags);
249    if (!rt_mutex_owner(lock)) {
250        /*
251         * If the requeue above changed the top waiter, then we need
252         * to wake the new top waiter up to try to get the lock.
253         */
254
255        if (top_waiter != rt_mutex_top_waiter(lock))
256            wake_up_process(rt_mutex_top_waiter(lock)->task);
257        raw_spin_unlock(&lock->wait_lock);
258        goto out_put_task;
259    }
260    put_task_struct(task);
261
262    /* Grab the next task */
263    task = rt_mutex_owner(lock);
264    get_task_struct(task);
265    raw_spin_lock_irqsave(&task->pi_lock, flags);
266
267    if (waiter == rt_mutex_top_waiter(lock)) {
268        /* Boost the owner */
269        plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
270        waiter->pi_list_entry.prio = waiter->list_entry.prio;
271        plist_add(&waiter->pi_list_entry, &task->pi_waiters);
272        __rt_mutex_adjust_prio(task);
273
274    } else if (top_waiter == waiter) {
275        /* Deboost the owner */
276        plist_del(&waiter->pi_list_entry, &task->pi_waiters);
277        waiter = rt_mutex_top_waiter(lock);
278        waiter->pi_list_entry.prio = waiter->list_entry.prio;
279        plist_add(&waiter->pi_list_entry, &task->pi_waiters);
280        __rt_mutex_adjust_prio(task);
281    }
282
283    raw_spin_unlock_irqrestore(&task->pi_lock, flags);
284
285    top_waiter = rt_mutex_top_waiter(lock);
286    raw_spin_unlock(&lock->wait_lock);
287
288    if (!detect_deadlock && waiter != top_waiter)
289        goto out_put_task;
290
291    goto again;
292
293 out_unlock_pi:
294    raw_spin_unlock_irqrestore(&task->pi_lock, flags);
295 out_put_task:
296    put_task_struct(task);
297
298    return ret;
299}
300
301/*
302 * Try to take an rt-mutex
303 *
304 * Must be called with lock->wait_lock held.
305 *
306 * @lock: the lock to be acquired.
307 * @task: the task which wants to acquire the lock
308 * @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
309 */
310static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
311        struct rt_mutex_waiter *waiter)
312{
313    /*
314     * We have to be careful here if the atomic speedups are
315     * enabled, such that, when
316     * - no other waiter is on the lock
317     * - the lock has been released since we did the cmpxchg
318     * the lock can be released or taken while we are doing the
319     * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
320     *
321     * The atomic acquire/release aware variant of
322     * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
323     * the WAITERS bit, the atomic release / acquire can not
324     * happen anymore and lock->wait_lock protects us from the
325     * non-atomic case.
326     *
327     * Note, that this might set lock->owner =
328     * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
329     * any more. This is fixed up when we take the ownership.
330     * This is the transitional state explained at the top of this file.
331     */
332    mark_rt_mutex_waiters(lock);
333
334    if (rt_mutex_owner(lock))
335        return 0;
336
337    /*
338     * It will get the lock because of one of these conditions:
339     * 1) there is no waiter
340     * 2) higher priority than waiters
341     * 3) it is top waiter
342     */
343    if (rt_mutex_has_waiters(lock)) {
344        if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
345            if (!waiter || waiter != rt_mutex_top_waiter(lock))
346                return 0;
347        }
348    }
349
350    if (waiter || rt_mutex_has_waiters(lock)) {
351        unsigned long flags;
352        struct rt_mutex_waiter *top;
353
354        raw_spin_lock_irqsave(&task->pi_lock, flags);
355
356        /* remove the queued waiter. */
357        if (waiter) {
358            plist_del(&waiter->list_entry, &lock->wait_list);
359            task->pi_blocked_on = NULL;
360        }
361
362        /*
363         * We have to enqueue the top waiter(if it exists) into
364         * task->pi_waiters list.
365         */
366        if (rt_mutex_has_waiters(lock)) {
367            top = rt_mutex_top_waiter(lock);
368            top->pi_list_entry.prio = top->list_entry.prio;
369            plist_add(&top->pi_list_entry, &task->pi_waiters);
370        }
371        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
372    }
373
374    /* We got the lock. */
375    debug_rt_mutex_lock(lock);
376
377    rt_mutex_set_owner(lock, task);
378
379    rt_mutex_deadlock_account_lock(lock, task);
380
381    return 1;
382}
383
384/*
385 * Task blocks on lock.
386 *
387 * Prepare waiter and propagate pi chain
388 *
389 * This must be called with lock->wait_lock held.
390 */
391static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
392                   struct rt_mutex_waiter *waiter,
393                   struct task_struct *task,
394                   int detect_deadlock)
395{
396    struct task_struct *owner = rt_mutex_owner(lock);
397    struct rt_mutex_waiter *top_waiter = waiter;
398    unsigned long flags;
399    int chain_walk = 0, res;
400
401    raw_spin_lock_irqsave(&task->pi_lock, flags);
402    __rt_mutex_adjust_prio(task);
403    waiter->task = task;
404    waiter->lock = lock;
405    plist_node_init(&waiter->list_entry, task->prio);
406    plist_node_init(&waiter->pi_list_entry, task->prio);
407
408    /* Get the top priority waiter on the lock */
409    if (rt_mutex_has_waiters(lock))
410        top_waiter = rt_mutex_top_waiter(lock);
411    plist_add(&waiter->list_entry, &lock->wait_list);
412
413    task->pi_blocked_on = waiter;
414
415    raw_spin_unlock_irqrestore(&task->pi_lock, flags);
416
417    if (!owner)
418        return 0;
419
420    if (waiter == rt_mutex_top_waiter(lock)) {
421        raw_spin_lock_irqsave(&owner->pi_lock, flags);
422        plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
423        plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
424
425        __rt_mutex_adjust_prio(owner);
426        if (owner->pi_blocked_on)
427            chain_walk = 1;
428        raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
429    }
430    else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
431        chain_walk = 1;
432
433    if (!chain_walk)
434        return 0;
435
436    /*
437     * The owner can't disappear while holding a lock,
438     * so the owner struct is protected by wait_lock.
439     * Gets dropped in rt_mutex_adjust_prio_chain()!
440     */
441    get_task_struct(owner);
442
443    raw_spin_unlock(&lock->wait_lock);
444
445    res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
446                     task);
447
448    raw_spin_lock(&lock->wait_lock);
449
450    return res;
451}
452
453/*
454 * Wake up the next waiter on the lock.
455 *
456 * Remove the top waiter from the current tasks waiter list and wake it up.
457 *
458 * Called with lock->wait_lock held.
459 */
460static void wakeup_next_waiter(struct rt_mutex *lock)
461{
462    struct rt_mutex_waiter *waiter;
463    unsigned long flags;
464
465    raw_spin_lock_irqsave(&current->pi_lock, flags);
466
467    waiter = rt_mutex_top_waiter(lock);
468
469    /*
470     * Remove it from current->pi_waiters. We do not adjust a
471     * possible priority boost right now. We execute wakeup in the
472     * boosted mode and go back to normal after releasing
473     * lock->wait_lock.
474     */
475    plist_del(&waiter->pi_list_entry, &current->pi_waiters);
476
477    rt_mutex_set_owner(lock, NULL);
478
479    raw_spin_unlock_irqrestore(&current->pi_lock, flags);
480
481    wake_up_process(waiter->task);
482}
483
484/*
485 * Remove a waiter from a lock and give up
486 *
487 * Must be called with lock->wait_lock held and
488 * have just failed to try_to_take_rt_mutex().
489 */
490static void remove_waiter(struct rt_mutex *lock,
491              struct rt_mutex_waiter *waiter)
492{
493    int first = (waiter == rt_mutex_top_waiter(lock));
494    struct task_struct *owner = rt_mutex_owner(lock);
495    unsigned long flags;
496    int chain_walk = 0;
497
498    raw_spin_lock_irqsave(&current->pi_lock, flags);
499    plist_del(&waiter->list_entry, &lock->wait_list);
500    current->pi_blocked_on = NULL;
501    raw_spin_unlock_irqrestore(&current->pi_lock, flags);
502
503    if (!owner)
504        return;
505
506    if (first) {
507
508        raw_spin_lock_irqsave(&owner->pi_lock, flags);
509
510        plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
511
512        if (rt_mutex_has_waiters(lock)) {
513            struct rt_mutex_waiter *next;
514
515            next = rt_mutex_top_waiter(lock);
516            plist_add(&next->pi_list_entry, &owner->pi_waiters);
517        }
518        __rt_mutex_adjust_prio(owner);
519
520        if (owner->pi_blocked_on)
521            chain_walk = 1;
522
523        raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
524    }
525
526    WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
527
528    if (!chain_walk)
529        return;
530
531    /* gets dropped in rt_mutex_adjust_prio_chain()! */
532    get_task_struct(owner);
533
534    raw_spin_unlock(&lock->wait_lock);
535
536    rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
537
538    raw_spin_lock(&lock->wait_lock);
539}
540
541/*
542 * Recheck the pi chain, in case we got a priority setting
543 *
544 * Called from sched_setscheduler
545 */
546void rt_mutex_adjust_pi(struct task_struct *task)
547{
548    struct rt_mutex_waiter *waiter;
549    unsigned long flags;
550
551    raw_spin_lock_irqsave(&task->pi_lock, flags);
552
553    waiter = task->pi_blocked_on;
554    if (!waiter || waiter->list_entry.prio == task->prio) {
555        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
556        return;
557    }
558
559    raw_spin_unlock_irqrestore(&task->pi_lock, flags);
560
561    /* gets dropped in rt_mutex_adjust_prio_chain()! */
562    get_task_struct(task);
563    rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
564}
565
566/**
567 * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
568 * @lock: the rt_mutex to take
569 * @state: the state the task should block in (TASK_INTERRUPTIBLE
570 * or TASK_UNINTERRUPTIBLE)
571 * @timeout: the pre-initialized and started timer, or NULL for none
572 * @waiter: the pre-initialized rt_mutex_waiter
573 *
574 * lock->wait_lock must be held by the caller.
575 */
576static int __sched
577__rt_mutex_slowlock(struct rt_mutex *lock, int state,
578            struct hrtimer_sleeper *timeout,
579            struct rt_mutex_waiter *waiter)
580{
581    int ret = 0;
582
583    for (;;) {
584        /* Try to acquire the lock: */
585        if (try_to_take_rt_mutex(lock, current, waiter))
586            break;
587
588        /*
589         * TASK_INTERRUPTIBLE checks for signals and
590         * timeout. Ignored otherwise.
591         */
592        if (unlikely(state == TASK_INTERRUPTIBLE)) {
593            /* Signal pending? */
594            if (signal_pending(current))
595                ret = -EINTR;
596            if (timeout && !timeout->task)
597                ret = -ETIMEDOUT;
598            if (ret)
599                break;
600        }
601
602        raw_spin_unlock(&lock->wait_lock);
603
604        debug_rt_mutex_print_deadlock(waiter);
605
606        schedule_rt_mutex(lock);
607
608        raw_spin_lock(&lock->wait_lock);
609        set_current_state(state);
610    }
611
612    return ret;
613}
614
615/*
616 * Slow path lock function:
617 */
618static int __sched
619rt_mutex_slowlock(struct rt_mutex *lock, int state,
620          struct hrtimer_sleeper *timeout,
621          int detect_deadlock)
622{
623    struct rt_mutex_waiter waiter;
624    int ret = 0;
625
626    debug_rt_mutex_init_waiter(&waiter);
627
628    raw_spin_lock(&lock->wait_lock);
629
630    /* Try to acquire the lock again: */
631    if (try_to_take_rt_mutex(lock, current, NULL)) {
632        raw_spin_unlock(&lock->wait_lock);
633        return 0;
634    }
635
636    set_current_state(state);
637
638    /* Setup the timer, when timeout != NULL */
639    if (unlikely(timeout)) {
640        hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
641        if (!hrtimer_active(&timeout->timer))
642            timeout->task = NULL;
643    }
644
645    ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
646
647    if (likely(!ret))
648        ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
649
650    set_current_state(TASK_RUNNING);
651
652    if (unlikely(ret))
653        remove_waiter(lock, &waiter);
654
655    /*
656     * try_to_take_rt_mutex() sets the waiter bit
657     * unconditionally. We might have to fix that up.
658     */
659    fixup_rt_mutex_waiters(lock);
660
661    raw_spin_unlock(&lock->wait_lock);
662
663    /* Remove pending timer: */
664    if (unlikely(timeout))
665        hrtimer_cancel(&timeout->timer);
666
667    debug_rt_mutex_free_waiter(&waiter);
668
669    return ret;
670}
671
672/*
673 * Slow path try-lock function:
674 */
675static inline int
676rt_mutex_slowtrylock(struct rt_mutex *lock)
677{
678    int ret = 0;
679
680    raw_spin_lock(&lock->wait_lock);
681
682    if (likely(rt_mutex_owner(lock) != current)) {
683
684        ret = try_to_take_rt_mutex(lock, current, NULL);
685        /*
686         * try_to_take_rt_mutex() sets the lock waiters
687         * bit unconditionally. Clean this up.
688         */
689        fixup_rt_mutex_waiters(lock);
690    }
691
692    raw_spin_unlock(&lock->wait_lock);
693
694    return ret;
695}
696
697/*
698 * Slow path to release a rt-mutex:
699 */
700static void __sched
701rt_mutex_slowunlock(struct rt_mutex *lock)
702{
703    raw_spin_lock(&lock->wait_lock);
704
705    debug_rt_mutex_unlock(lock);
706
707    rt_mutex_deadlock_account_unlock(current);
708
709    if (!rt_mutex_has_waiters(lock)) {
710        lock->owner = NULL;
711        raw_spin_unlock(&lock->wait_lock);
712        return;
713    }
714
715    wakeup_next_waiter(lock);
716
717    raw_spin_unlock(&lock->wait_lock);
718
719    /* Undo pi boosting if necessary: */
720    rt_mutex_adjust_prio(current);
721}
722
723/*
724 * debug aware fast / slowpath lock,trylock,unlock
725 *
726 * The atomic acquire/release ops are compiled away, when either the
727 * architecture does not support cmpxchg or when debugging is enabled.
728 */
729static inline int
730rt_mutex_fastlock(struct rt_mutex *lock, int state,
731          int detect_deadlock,
732          int (*slowfn)(struct rt_mutex *lock, int state,
733                struct hrtimer_sleeper *timeout,
734                int detect_deadlock))
735{
736    if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
737        rt_mutex_deadlock_account_lock(lock, current);
738        return 0;
739    } else
740        return slowfn(lock, state, NULL, detect_deadlock);
741}
742
743static inline int
744rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
745            struct hrtimer_sleeper *timeout, int detect_deadlock,
746            int (*slowfn)(struct rt_mutex *lock, int state,
747                      struct hrtimer_sleeper *timeout,
748                      int detect_deadlock))
749{
750    if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
751        rt_mutex_deadlock_account_lock(lock, current);
752        return 0;
753    } else
754        return slowfn(lock, state, timeout, detect_deadlock);
755}
756
757static inline int
758rt_mutex_fasttrylock(struct rt_mutex *lock,
759             int (*slowfn)(struct rt_mutex *lock))
760{
761    if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
762        rt_mutex_deadlock_account_lock(lock, current);
763        return 1;
764    }
765    return slowfn(lock);
766}
767
768static inline void
769rt_mutex_fastunlock(struct rt_mutex *lock,
770            void (*slowfn)(struct rt_mutex *lock))
771{
772    if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
773        rt_mutex_deadlock_account_unlock(current);
774    else
775        slowfn(lock);
776}
777
778/**
779 * rt_mutex_lock - lock a rt_mutex
780 *
781 * @lock: the rt_mutex to be locked
782 */
783void __sched rt_mutex_lock(struct rt_mutex *lock)
784{
785    might_sleep();
786
787    rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
788}
789EXPORT_SYMBOL_GPL(rt_mutex_lock);
790
791/**
792 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
793 *
794 * @lock: the rt_mutex to be locked
795 * @detect_deadlock: deadlock detection on/off
796 *
797 * Returns:
798 * 0 on success
799 * -EINTR when interrupted by a signal
800 * -EDEADLK when the lock would deadlock (when deadlock detection is on)
801 */
802int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
803                         int detect_deadlock)
804{
805    might_sleep();
806
807    return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
808                 detect_deadlock, rt_mutex_slowlock);
809}
810EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
811
812/**
813 * rt_mutex_timed_lock - lock a rt_mutex interruptible
814 * the timeout structure is provided
815 * by the caller
816 *
817 * @lock: the rt_mutex to be locked
818 * @timeout: timeout structure or NULL (no timeout)
819 * @detect_deadlock: deadlock detection on/off
820 *
821 * Returns:
822 * 0 on success
823 * -EINTR when interrupted by a signal
824 * -ETIMEDOUT when the timeout expired
825 * -EDEADLK when the lock would deadlock (when deadlock detection is on)
826 */
827int
828rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
829            int detect_deadlock)
830{
831    might_sleep();
832
833    return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
834                       detect_deadlock, rt_mutex_slowlock);
835}
836EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
837
838/**
839 * rt_mutex_trylock - try to lock a rt_mutex
840 *
841 * @lock: the rt_mutex to be locked
842 *
843 * Returns 1 on success and 0 on contention
844 */
845int __sched rt_mutex_trylock(struct rt_mutex *lock)
846{
847    return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
848}
849EXPORT_SYMBOL_GPL(rt_mutex_trylock);
850
851/**
852 * rt_mutex_unlock - unlock a rt_mutex
853 *
854 * @lock: the rt_mutex to be unlocked
855 */
856void __sched rt_mutex_unlock(struct rt_mutex *lock)
857{
858    rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
859}
860EXPORT_SYMBOL_GPL(rt_mutex_unlock);
861
862/**
863 * rt_mutex_destroy - mark a mutex unusable
864 * @lock: the mutex to be destroyed
865 *
866 * This function marks the mutex uninitialized, and any subsequent
867 * use of the mutex is forbidden. The mutex must not be locked when
868 * this function is called.
869 */
870void rt_mutex_destroy(struct rt_mutex *lock)
871{
872    WARN_ON(rt_mutex_is_locked(lock));
873#ifdef CONFIG_DEBUG_RT_MUTEXES
874    lock->magic = NULL;
875#endif
876}
877
878EXPORT_SYMBOL_GPL(rt_mutex_destroy);
879
880/**
881 * __rt_mutex_init - initialize the rt lock
882 *
883 * @lock: the rt lock to be initialized
884 *
885 * Initialize the rt lock to unlocked state.
886 *
887 * Initializing of a locked rt lock is not allowed
888 */
889void __rt_mutex_init(struct rt_mutex *lock, const char *name)
890{
891    lock->owner = NULL;
892    raw_spin_lock_init(&lock->wait_lock);
893    plist_head_init(&lock->wait_list);
894
895    debug_rt_mutex_init(lock, name);
896}
897EXPORT_SYMBOL_GPL(__rt_mutex_init);
898
899/**
900 * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
901 * proxy owner
902 *
903 * @lock: the rt_mutex to be locked
904 * @proxy_owner:the task to set as owner
905 *
906 * No locking. Caller has to do serializing itself
907 * Special API call for PI-futex support
908 */
909void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
910                struct task_struct *proxy_owner)
911{
912    __rt_mutex_init(lock, NULL);
913    debug_rt_mutex_proxy_lock(lock, proxy_owner);
914    rt_mutex_set_owner(lock, proxy_owner);
915    rt_mutex_deadlock_account_lock(lock, proxy_owner);
916}
917
918/**
919 * rt_mutex_proxy_unlock - release a lock on behalf of owner
920 *
921 * @lock: the rt_mutex to be locked
922 *
923 * No locking. Caller has to do serializing itself
924 * Special API call for PI-futex support
925 */
926void rt_mutex_proxy_unlock(struct rt_mutex *lock,
927               struct task_struct *proxy_owner)
928{
929    debug_rt_mutex_proxy_unlock(lock);
930    rt_mutex_set_owner(lock, NULL);
931    rt_mutex_deadlock_account_unlock(proxy_owner);
932}
933
934/**
935 * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
936 * @lock: the rt_mutex to take
937 * @waiter: the pre-initialized rt_mutex_waiter
938 * @task: the task to prepare
939 * @detect_deadlock: perform deadlock detection (1) or not (0)
940 *
941 * Returns:
942 * 0 - task blocked on lock
943 * 1 - acquired the lock for task, caller should wake it up
944 * <0 - error
945 *
946 * Special API call for FUTEX_REQUEUE_PI support.
947 */
948int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
949                  struct rt_mutex_waiter *waiter,
950                  struct task_struct *task, int detect_deadlock)
951{
952    int ret;
953
954    raw_spin_lock(&lock->wait_lock);
955
956    if (try_to_take_rt_mutex(lock, task, NULL)) {
957        raw_spin_unlock(&lock->wait_lock);
958        return 1;
959    }
960
961    ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
962
963    if (ret && !rt_mutex_owner(lock)) {
964        /*
965         * Reset the return value. We might have
966         * returned with -EDEADLK and the owner
967         * released the lock while we were walking the
968         * pi chain. Let the waiter sort it out.
969         */
970        ret = 0;
971    }
972
973    if (unlikely(ret))
974        remove_waiter(lock, waiter);
975
976    raw_spin_unlock(&lock->wait_lock);
977
978    debug_rt_mutex_print_deadlock(waiter);
979
980    return ret;
981}
982
983/**
984 * rt_mutex_next_owner - return the next owner of the lock
985 *
986 * @lock: the rt lock query
987 *
988 * Returns the next owner of the lock or NULL
989 *
990 * Caller has to serialize against other accessors to the lock
991 * itself.
992 *
993 * Special API call for PI-futex support
994 */
995struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
996{
997    if (!rt_mutex_has_waiters(lock))
998        return NULL;
999
1000    return rt_mutex_top_waiter(lock)->task;
1001}
1002
1003/**
1004 * rt_mutex_finish_proxy_lock() - Complete lock acquisition
1005 * @lock: the rt_mutex we were woken on
1006 * @to: the timeout, null if none. hrtimer should already have
1007 * been started.
1008 * @waiter: the pre-initialized rt_mutex_waiter
1009 * @detect_deadlock: perform deadlock detection (1) or not (0)
1010 *
1011 * Complete the lock acquisition started our behalf by another thread.
1012 *
1013 * Returns:
1014 * 0 - success
1015 * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK
1016 *
1017 * Special API call for PI-futex requeue support
1018 */
1019int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
1020                   struct hrtimer_sleeper *to,
1021                   struct rt_mutex_waiter *waiter,
1022                   int detect_deadlock)
1023{
1024    int ret;
1025
1026    raw_spin_lock(&lock->wait_lock);
1027
1028    set_current_state(TASK_INTERRUPTIBLE);
1029
1030    ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
1031
1032    set_current_state(TASK_RUNNING);
1033
1034    if (unlikely(ret))
1035        remove_waiter(lock, waiter);
1036
1037    /*
1038     * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
1039     * have to fix that up.
1040     */
1041    fixup_rt_mutex_waiters(lock);
1042
1043    raw_spin_unlock(&lock->wait_lock);
1044
1045    return ret;
1046}
1047

Archive Download this file



interactive