From: Ingo Molnar Date: Tue, 6 Jan 2009 08:32:11 +0000 (+0100) Subject: Merge branches 'core/futexes', 'core/locking', 'core/rcu' and 'linus' into core/urgent X-Git-Tag: v2.6.29-rc1~226^2 X-Git-Url: http://pilppa.com/gitweb/?a=commitdiff_plain;h=fdbc0450df12cc9cb397f3497db4b0cad7c1a8ff;p=linux-2.6-omap-h63xx.git Merge branches 'core/futexes', 'core/locking', 'core/rcu' and 'linus' into core/urgent --- fdbc0450df12cc9cb397f3497db4b0cad7c1a8ff diff --cc kernel/futex.c index 7c6cbabe52b,206d4c90688,8af10027514,7c6cbabe52b..002aa189eb0 --- a/kernel/futex.c +++ b/kernel/futex.c @@@@@ -144,45 -144,48 -161,6 -144,45 +144,48 @@@@@ static inline int match_futex(union fut && key1->both.offset == key2->both.offset); } + /* + * Take a reference to the resource addressed by a key. + * Can be called while holding spinlocks. + * + */ + static void get_futex_key_refs(union futex_key *key) + { + if (!key->both.ptr) + return; + + switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { + case FUT_OFF_INODE: + atomic_inc(&key->shared.inode->i_count); + break; + case FUT_OFF_MMSHARED: + atomic_inc(&key->private.mm->mm_count); + break; + } + } + + /* + * Drop a reference to the resource addressed by a key. + * The hash bucket spinlock must not be held. + */ + static void drop_futex_key_refs(union futex_key *key) + { - - if (!key->both.ptr) + ++ if (!key->both.ptr) { + ++ /* If we're here then we tried to put a key we failed to get */ + ++ WARN_ON_ONCE(1); + return; + ++ } + + switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { + case FUT_OFF_INODE: + iput(key->shared.inode); + break; + case FUT_OFF_MMSHARED: + mmdrop(key->private.mm); + break; + } + } + /** * get_futex_key - Get parameters which are the keys for a futex. * @uaddr: virtual address of the futex @@@@@ -730,8 -726,8 -759,8 -730,8 +733,8 @@@@@ static int futex_wake(u32 __user *uaddr } spin_unlock(&hb->lock); - -out: + put_futex_key(fshared, &key); + +out: - futex_unlock_mm(fshared); return ret; } @@@@@ -794,9 -790,9 -826,9 -794,9 +797,9 @@@@@ retry */ if (attempt++) { ret = futex_handle_fault((unsigned long)uaddr2, - fshared, attempt); + attempt); if (ret) - -- goto out; + ++ goto out_put_keys; goto retry; } @@@@@ -834,10 -830,11 -872,9 -834,10 +837,11 @@@@@ spin_unlock(&hb1->lock); if (hb1 != hb2) spin_unlock(&hb2->lock); - -out: + ++out_put_keys: + put_futex_key(fshared, &key2); + ++out_put_key1: + put_futex_key(fshared, &key1); - - + +out: - futex_unlock_mm(fshared); - return ret; } @@@@@ -854,7 -851,7 -892,9 -854,7 +858,7 @@@@@ static int futex_requeue(u32 __user *ua struct futex_q *this, *next; int ret, drop_count = 0; - -- retry: - futex_lock_mm(fshared); - + ++retry: ret = get_futex_key(uaddr1, fshared, &key1); if (unlikely(ret != 0)) goto out; @@@@@ -927,9 -924,11 -973,8 -927,9 +931,11 @@@@@ out_unlock while (--drop_count >= 0) drop_futex_key_refs(&key1); - -out: + ++out_put_keys: + put_futex_key(fshared, &key2); + ++out_put_key1: + put_futex_key(fshared, &key1); + +out: - futex_unlock_mm(fshared); return ret; } @@@@@ -1172,11 -1171,11 -1217,12 -1172,11 +1178,11 @@@@@ static int futex_wait(u32 __user *uaddr q.pi_state = NULL; q.bitset = bitset; - -- retry: - futex_lock_mm(fshared); - + ++retry: + q.key = FUTEX_KEY_INIT; ret = get_futex_key(uaddr, fshared, &q.key); if (unlikely(ret != 0)) - -- goto out_release_sem; + ++ goto out; hb = queue_lock(&q); @@@@@ -1204,6 -1203,7 -1250,12 -1204,6 +1210,7 @@@@@ if (unlikely(ret)) { queue_unlock(&q, hb); - - /* - * If we would have faulted, release mmap_sem, fault it in and - * start all over again. - */ - futex_unlock_mm(fshared); + ++ put_futex_key(fshared, &q.key); ret = get_user(uval, uaddr); @@@@@ -1305,11 -1305,11 -1359,11 -1305,11 +1312,11 @@@@@ return -ERESTART_RESTARTBLOCK; } - -- out_unlock_release_sem: + ++out_unlock_put_key: queue_unlock(&q, hb); - - - - out_release_sem: + put_futex_key(fshared, &q.key); + + - out_release_sem: - futex_unlock_mm(fshared); + ++out: return ret; } @@@@@ -1358,16 -1358,16 -1411,17 -1358,16 +1365,16 @@@@@ static int futex_lock_pi(u32 __user *ua } q.pi_state = NULL; - -- retry: - futex_lock_mm(fshared); - + ++retry: + q.key = FUTEX_KEY_INIT; ret = get_futex_key(uaddr, fshared, &q.key); if (unlikely(ret != 0)) - -- goto out_release_sem; + ++ goto out; - -- retry_unlocked: + ++retry_unlocked: hb = queue_lock(&q); - -- retry_locked: + ++retry_locked: ret = lock_taken = 0; /* @@@@@ -1561,29 -1561,30 -1624,31 -1561,29 +1568,30 @@@@@ destroy_hrtimer_on_stack(&to->timer); return ret != -EINTR ? ret : -ERESTARTNOINTR; - -- out_unlock_release_sem: + ++out_unlock_put_key: queue_unlock(&q, hb); - -- out_release_sem: - futex_unlock_mm(fshared); + ++out_put_key: + put_futex_key(fshared, &q.key); + ++out: if (to) destroy_hrtimer_on_stack(&to->timer); return ret; - -- uaddr_faulted: + ++uaddr_faulted: /* - * We have to r/w *(int __user *)uaddr, but we can't modify it - * non-atomically. Therefore, if get_user below is not - * enough, we need to handle the fault ourselves, while - * still holding the mmap_sem. - * - * ... and hb->lock. :-) --ANK + * We have to r/w *(int __user *)uaddr, and we have to modify it + * atomically. Therefore, if we continue to fault after get_user() + * below, we need to handle the fault ourselves, while still holding + * the mmap_sem. This can occur if the uaddr is under contention as + * we have to drop the mmap_sem in order to call get_user(). */ queue_unlock(&q, hb); if (attempt++) { - ret = futex_handle_fault((unsigned long)uaddr, fshared, - attempt); + ret = futex_handle_fault((unsigned long)uaddr, attempt); if (ret) - -- goto out_release_sem; + ++ goto out_put_key; goto retry_unlocked; } @@@@@ -1675,9 -1676,9 -1746,9 -1675,9 +1683,9 @@@@@ retry_unlocked out_unlock: spin_unlock(&hb->lock); - --out: - futex_unlock_mm(fshared); + put_futex_key(fshared, &key); + ++out: return ret; pi_faulted: