x86: some lock annotations for user copy paths, v2

 - introduce might_fault()
 - handle the atomic user copy paths correctly

[ mingo@elte.hu: move might_sleep() outside of in_atomic(). ]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 8eedde2a9cac614f97580d61e03b97662ad3e762..7393152a252e74bf0992993e7d3ac58b887de763 100644 (file)
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -32,9 +32,7 @@ static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned lon
 #define __do_strncpy_from_user(dst, src, count, res)                      \
 do {                                                                      \
        int __d0, __d1, __d2;                                              \
-       might_sleep();                                                     \
-       if (current->mm)                                                   \
-               might_lock_read(&current->mm->mmap_sem);                   \
+       might_fault();                                                     \
        __asm__ __volatile__(                                              \
                "       testl %1,%1\n"                                     \
                "       jz 2f\n"                                           \
@@ -121,9 +119,7 @@ EXPORT_SYMBOL(strncpy_from_user);
 #define __do_clear_user(addr,size)                                     \
 do {                                                                   \
        int __d0;                                                       \
-       might_sleep();                                                  \
-       if (current->mm)                                                \
-               might_lock_read(&current->mm->mmap_sem);                \
+       might_fault();                                                  \
        __asm__ __volatile__(                                           \
                "0:     rep; stosl\n"                                   \
                "       movl %2,%0\n"                                   \
@@ -193,9 +189,7 @@ long strnlen_user(const char __user *s, long n)
        unsigned long mask = -__addr_ok(s);
        unsigned long res, tmp;
 
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
 
        __asm__ __volatile__(
                "       testl %0, %0\n"

diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 847d12945998a99c3bca081aa89a56940b911621..64d6c84e6353e9d50ce3d7f7cda6ee7178ca3494 100644 (file)
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -15,9 +15,7 @@
 #define __do_strncpy_from_user(dst,src,count,res)                         \
 do {                                                                      \
        long __d0, __d1, __d2;                                             \
-       might_sleep();                                                     \
-       if (current->mm)                                                   \
-               might_lock_read(&current->mm->mmap_sem);                   \
+       might_fault();                                                     \
        __asm__ __volatile__(                                              \
                "       testq %1,%1\n"                                     \
                "       jz 2f\n"                                           \
@@ -66,9 +64,7 @@ EXPORT_SYMBOL(strncpy_from_user);
 unsigned long __clear_user(void __user *addr, unsigned long size)
 {
        long __d0;
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
        /* no memory constraint because it doesn't change any memory gcc knows
           about */
        asm volatile(

diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h
index ad29752a171338a94a6e57a437ebb7bfd9e62ecd..39f8420c75d922fea19988ec055e0aa4295bef99 100644 (file)
--- a/include/asm-x86/uaccess.h
+++ b/include/asm-x86/uaccess.h
@@ -8,8 +8,6 @@
 #include <linux/thread_info.h>
 #include <linux/prefetch.h>
 #include <linux/string.h>
-#include <linux/lockdep.h>
-#include <linux/sched.h>
 #include <asm/asm.h>
 #include <asm/page.h>
 
@@ -159,9 +157,7 @@ extern int __get_user_bad(void);
        int __ret_gu;                                                   \
        unsigned long __val_gu;                                         \
        __chk_user_ptr(ptr);                                            \
-       might_sleep();                                                  \
-       if (current->mm)                                                \
-               might_lock_read(&current->mm->mmap_sem);                \
+       might_fault();                                                  \
        switch (sizeof(*(ptr))) {                                       \
        case 1:                                                         \
                __get_user_x(1, __ret_gu, __val_gu, ptr);               \
@@ -246,9 +242,7 @@ extern void __put_user_8(void);
        int __ret_pu;                                           \
        __typeof__(*(ptr)) __pu_val;                            \
        __chk_user_ptr(ptr);                                    \
-       might_sleep();                                          \
-       if (current->mm)                                        \
-               might_lock_read(&current->mm->mmap_sem);        \
+       might_fault();                                          \
        __pu_val = x;                                           \
        switch (sizeof(*(ptr))) {                               \
        case 1:                                                 \
@@ -273,9 +267,7 @@ extern void __put_user_8(void);
 #define __put_user_size(x, ptr, size, retval, errret)                  \
 do {                                                                   \
        retval = 0;                                                     \
-       might_sleep();                                                  \
-       if (current->mm)                                                \
-               might_lock_read(&current->mm->mmap_sem);                \
+       might_fault();                                                  \
        __chk_user_ptr(ptr);                                            \
        switch (size) {                                                 \
        case 1:                                                         \
@@ -328,9 +320,7 @@ do {                                                                        \
 #define __get_user_size(x, ptr, size, retval, errret)                  \
 do {                                                                   \
        retval = 0;                                                     \
-       might_sleep();                                                  \
-       if (current->mm)                                                \
-               might_lock_read(&current->mm->mmap_sem);                \
+       might_fault();                                                  \
        __chk_user_ptr(ptr);                                            \
        switch (size) {                                                 \
        case 1:                                                         \

diff --git a/include/asm-x86/uaccess_32.h b/include/asm-x86/uaccess_32.h
index d725e2d703f7dfa706ed9da3f31053253cb88ffc..d10e842ec3eed2d9cc97fcee885b2cc9be17a0b5 100644 (file)
--- a/include/asm-x86/uaccess_32.h
+++ b/include/asm-x86/uaccess_32.h
@@ -82,9 +82,7 @@ __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
 static __always_inline unsigned long __must_check
 __copy_to_user(void __user *to, const void *from, unsigned long n)
 {
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
        return __copy_to_user_inatomic(to, from, n);
 }
 
@@ -139,9 +137,7 @@ __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
 static __always_inline unsigned long
 __copy_from_user(void *to, const void __user *from, unsigned long n)
 {
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
        if (__builtin_constant_p(n)) {
                unsigned long ret;
 
@@ -163,9 +159,7 @@ __copy_from_user(void *to, const void __user *from, unsigned long n)
 static __always_inline unsigned long __copy_from_user_nocache(void *to,
                                const void __user *from, unsigned long n)
 {
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
        if (__builtin_constant_p(n)) {
                unsigned long ret;
 

diff --git a/include/asm-x86/uaccess_64.h b/include/asm-x86/uaccess_64.h
index 40a7205fe576744135cb4210b4dcce5634f9874b..13fd56fbc3aba9791d0fc5b9655f311eb4c8436c 100644 (file)
--- a/include/asm-x86/uaccess_64.h
+++ b/include/asm-x86/uaccess_64.h
@@ -29,9 +29,7 @@ int __copy_from_user(void *dst, const void __user *src, unsigned size)
 {
        int ret = 0;
 
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
        if (!__builtin_constant_p(size))
                return copy_user_generic(dst, (__force void *)src, size);
        switch (size) {
@@ -75,9 +73,7 @@ int __copy_to_user(void __user *dst, const void *src, unsigned size)
 {
        int ret = 0;
 
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
        if (!__builtin_constant_p(size))
                return copy_user_generic((__force void *)dst, src, size);
        switch (size) {
@@ -121,9 +117,7 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
 {
        int ret = 0;
 
-       might_sleep();
-       if (current->mm)
-               might_lock_read(&current->mm->mmap_sem);
+       might_fault();
        if (!__builtin_constant_p(size))
                return copy_user_generic((__force void *)dst,
                                         (__force void *)src, size);

diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 2651f805ba6d771b9ec1f26078609aebdb198853..e580ec095765ed1041e9fa7c61c2c527b5826744 100644 (file)
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -140,6 +140,15 @@ extern int _cond_resched(void);
                (__x < 0) ? -__x : __x;         \
        })
 
+#ifdef CONFIG_PROVE_LOCKING
+void might_fault(void);
+#else
+static inline void might_fault(void)
+{
+       might_sleep();
+}
+#endif
+
 extern struct atomic_notifier_head panic_notifier_list;
 extern long (*panic_blink)(long time);
 NORET_TYPE void panic(const char * fmt, ...)

diff --git a/mm/memory.c b/mm/memory.c
index 1002f473f497c37f34c5c33d4dfa7d2bd49faed0..b8fdf4e5e65b52a812413ffc4797edf4e056f42c 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3016,3 +3016,18 @@ void print_vma_addr(char *prefix, unsigned long ip)
        }
        up_read(&current->mm->mmap_sem);
 }
+
+#ifdef CONFIG_PROVE_LOCKING
+void might_fault(void)
+{
+       might_sleep();
+       /*
+        * it would be nicer only to annotate paths which are not under
+        * pagefault_disable, however that requires a larger audit and
+        * providing helpers like get_user_atomic.
+        */
+       if (!in_atomic() && current->mm)
+               might_lock_read(&current->mm->mmap_sem);
+}
+EXPORT_SYMBOL(might_fault);
+#endif