[PATCH] Don't use lock section for mutexes and semaphores

Lock sections cannot be handled by the dwarf2 unwinder.

Disadvantage is a taken branch in the hot path.

Signed-off-by: Andi Kleen <ak@suse.de>

diff --git a/include/asm-x86_64/mutex.h b/include/asm-x86_64/mutex.h
index 06fab6de2a88bc7395a9f68a76c502291b7e3e9d..16396b1de3e44e2068021f446d8b56f3b6f79448 100644 (file)
--- a/include/asm-x86_64/mutex.h
+++ b/include/asm-x86_64/mutex.h
@@ -25,13 +25,9 @@ do {                                                                 \
                                                                        \
        __asm__ __volatile__(                                           \
                LOCK_PREFIX "   decl (%%rdi)    \n"                     \
-                       "   js 2f               \n"                     \
-                       "1:                     \n"                     \
-                                                                       \
-               LOCK_SECTION_START("")                                  \
-                       "2: call "#fail_fn"     \n"                     \
-                       "   jmp 1b              \n"                     \
-               LOCK_SECTION_END                                        \
+                       "   jns 1f              \n"                     \
+                       "   call "#fail_fn"     \n"                     \
+                       "1:"                                            \
                                                                        \
                :"=D" (dummy)                                           \
                : "D" (v)                                               \
@@ -75,13 +71,9 @@ do {                                                                 \
                                                                        \
        __asm__ __volatile__(                                           \
                LOCK_PREFIX "   incl (%%rdi)    \n"                     \
-                       "   jle 2f              \n"                     \
-                       "1:                     \n"                     \
-                                                                       \
-               LOCK_SECTION_START("")                                  \
-                       "2: call "#fail_fn"     \n"                     \
-                       "   jmp 1b              \n"                     \
-               LOCK_SECTION_END                                        \
+                       "   jg 1f               \n"                     \
+                       "   call "#fail_fn"     \n"                     \
+                       "1:                       "                     \
                                                                        \
                :"=D" (dummy)                                           \
                : "D" (v)                                               \

diff --git a/include/asm-x86_64/semaphore.h b/include/asm-x86_64/semaphore.h
index 064df08b9a0f039f24a3cd302d10b480d0341c62..107bd90429e86e14e47185895d9ab222be3b0652 100644 (file)
--- a/include/asm-x86_64/semaphore.h
+++ b/include/asm-x86_64/semaphore.h
@@ -107,12 +107,9 @@ static inline void down(struct semaphore * sem)
        __asm__ __volatile__(
                "# atomic down operation\n\t"
                LOCK_PREFIX "decl %0\n\t"     /* --sem->count */
-               "js 2f\n"
-               "1:\n"
-               LOCK_SECTION_START("")
-               "2:\tcall __down_failed\n\t"
-               "jmp 1b\n"
-               LOCK_SECTION_END
+               "jns 1f\n\t"
+               "call __down_failed\n"
+               "1:"
                :"=m" (sem->count)
                :"D" (sem)
                :"memory");
@@ -130,14 +127,11 @@ static inline int down_interruptible(struct semaphore * sem)
 
        __asm__ __volatile__(
                "# atomic interruptible down operation\n\t"
+               "xorl %0,%0\n\t"
                LOCK_PREFIX "decl %1\n\t"     /* --sem->count */
-               "js 2f\n\t"
-               "xorl %0,%0\n"
-               "1:\n"
-               LOCK_SECTION_START("")
-               "2:\tcall __down_failed_interruptible\n\t"
-               "jmp 1b\n"
-               LOCK_SECTION_END
+               "jns 2f\n\t"
+               "call __down_failed_interruptible\n"
+               "2:\n"
                :"=a" (result), "=m" (sem->count)
                :"D" (sem)
                :"memory");
@@ -154,14 +148,11 @@ static inline int down_trylock(struct semaphore * sem)
 
        __asm__ __volatile__(
                "# atomic interruptible down operation\n\t"
+               "xorl %0,%0\n\t"
                LOCK_PREFIX "decl %1\n\t"     /* --sem->count */
-               "js 2f\n\t"
-               "xorl %0,%0\n"
-               "1:\n"
-               LOCK_SECTION_START("")
-               "2:\tcall __down_failed_trylock\n\t"
-               "jmp 1b\n"
-               LOCK_SECTION_END
+               "jns 2f\n\t"
+               "call __down_failed_trylock\n\t"
+               "2:\n"
                :"=a" (result), "=m" (sem->count)
                :"D" (sem)
                :"memory","cc");
@@ -179,12 +170,9 @@ static inline void up(struct semaphore * sem)
        __asm__ __volatile__(
                "# atomic up operation\n\t"
                LOCK_PREFIX "incl %0\n\t"     /* ++sem->count */
-               "jle 2f\n"
-               "1:\n"
-               LOCK_SECTION_START("")
-               "2:\tcall __up_wakeup\n\t"
-               "jmp 1b\n"
-               LOCK_SECTION_END
+               "jg 1f\n\t"
+               "call __up_wakeup\n"
+               "1:"
                :"=m" (sem->count)
                :"D" (sem)
                :"memory");

diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h
index a8e3d89f591f0a24c3464d562049dd0c78f4e8d5..1d0733b38a4b44115a6038ca911eac42d9594fed 100644 (file)
--- a/include/asm-x86_64/spinlock.h
+++ b/include/asm-x86_64/spinlock.h
@@ -22,14 +22,13 @@
 #define __raw_spin_lock_string \
        "\n1:\t" \
        LOCK_PREFIX " ; decl %0\n\t" \
-       "js 2f\n" \
-       LOCK_SECTION_START("") \
-       "2:\t" \
+       "jns 2f\n" \
+       "3:\n" \
        "rep;nop\n\t" \
        "cmpl $0,%0\n\t" \
-       "jle 2b\n\t" \
+       "jle 3b\n\t" \
        "jmp 1b\n" \
-       LOCK_SECTION_END
+       "2:\t" \
 
 #define __raw_spin_lock_string_up \
        "\n\tdecl %0"