[PATCH] Fix zeroing on exception in copy_*_user

- Don't zero for __copy_from_user_inatomic following i386.
This will prevent spurious zeros for parallel file system writers when
one does a exception
- The string instruction version didn't zero the output on
exception. Oops.

Also I cleaned up the code a bit while I was at it and added a minor
optimization to the string instruction path.

Signed-off-by: Andi Kleen <ak@suse.de>

diff --git a/arch/x86_64/kernel/x8664_ksyms.c b/arch/x86_64/kernel/x8664_ksyms.c
index 370952c4ff228b79dd5663c1df3de7d56737747a..c3454af5e3a25896ca6cac599c0e21ce10e73a26 100644 (file)
--- a/arch/x86_64/kernel/x8664_ksyms.c
+++ b/arch/x86_64/kernel/x8664_ksyms.c
@@ -29,6 +29,7 @@ EXPORT_SYMBOL(__put_user_8);
 EXPORT_SYMBOL(copy_user_generic);
 EXPORT_SYMBOL(copy_from_user);
 EXPORT_SYMBOL(copy_to_user);
+EXPORT_SYMBOL(__copy_from_user_inatomic);
 
 EXPORT_SYMBOL(copy_page);
 EXPORT_SYMBOL(clear_page);

diff --git a/arch/x86_64/lib/copy_user.S b/arch/x86_64/lib/copy_user.S
index 962f3a693c5ec6332db8eb1a4875d7a5c48f34d5..70bebd310408b98b531a721f7d83ca4ee97d0369 100644 (file)
--- a/arch/x86_64/lib/copy_user.S
+++ b/arch/x86_64/lib/copy_user.S
@@ -9,10 +9,29 @@
 
 #define FIX_ALIGNMENT 1
 
-       #include <asm/current.h>
-       #include <asm/asm-offsets.h>
-       #include <asm/thread_info.h>
-       #include <asm/cpufeature.h>
+#include <asm/current.h>
+#include <asm/asm-offsets.h>
+#include <asm/thread_info.h>
+#include <asm/cpufeature.h>
+
+       .macro ALTERNATIVE_JUMP feature,orig,alt
+0:
+       .byte 0xe9      /* 32bit jump */
+       .long \orig-1f  /* by default jump to orig */
+1:
+       .section .altinstr_replacement,"ax"
+2:     .byte 0xe9                   /* near jump with 32bit immediate */
+       .long \alt-1b /* offset */   /* or alternatively to alt */
+       .previous
+       .section .altinstructions,"a"
+       .align 8
+       .quad  0b
+       .quad  2b
+       .byte  \feature              /* when feature is set */
+       .byte  5
+       .byte  5
+       .previous
+       .endm
 
 /* Standard copy_to_user with segment limit checking */                
 ENTRY(copy_to_user)
@@ -23,25 +42,21 @@ ENTRY(copy_to_user)
        jc  bad_to_user
        cmpq threadinfo_addr_limit(%rax),%rcx
        jae bad_to_user
-2:
-       .byte 0xe9      /* 32bit jump */
-       .long .Lcug-1f
-1:
+       xorl %eax,%eax  /* clear zero flag */
+       ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
        CFI_ENDPROC
-ENDPROC(copy_to_user)
 
-       .section .altinstr_replacement,"ax"
-3:     .byte 0xe9                      /* replacement jmp with 32 bit immediate */
-       .long copy_user_generic_c-1b    /* offset */
-       .previous
-       .section .altinstructions,"a"
-       .align 8
-       .quad  2b
-       .quad  3b
-       .byte  X86_FEATURE_REP_GOOD
-       .byte  5
-       .byte  5
-       .previous
+ENTRY(copy_user_generic)
+       CFI_STARTPROC
+       movl $1,%ecx    /* set zero flag */
+       ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
+       CFI_ENDPROC
+
+ENTRY(__copy_from_user_inatomic)
+       CFI_STARTPROC
+       xorl %ecx,%ecx  /* clear zero flag */
+       ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
+       CFI_ENDPROC
 
 /* Standard copy_from_user with segment limit checking */      
 ENTRY(copy_from_user)
@@ -52,7 +67,8 @@ ENTRY(copy_from_user)
        jc  bad_from_user
        cmpq threadinfo_addr_limit(%rax),%rcx
        jae  bad_from_user
-       /* FALL THROUGH to copy_user_generic */
+       movl $1,%ecx    /* set zero flag */
+       ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
        CFI_ENDPROC
 ENDPROC(copy_from_user)
        
@@ -73,37 +89,26 @@ END(bad_from_user)
        
                
 /*
- * copy_user_generic - memory copy with exception handling.
+ * copy_user_generic_unrolled - memory copy with exception handling.
+ * This version is for CPUs like P4 that don't have efficient micro code for rep movsq
  *     
  * Input:      
  * rdi destination
  * rsi source
  * rdx count
+ * ecx zero flag -- if true zero destination on error
  *
  * Output:             
  * eax uncopied bytes or 0 if successful.
  */
-ENTRY(copy_user_generic)
+ENTRY(copy_user_generic_unrolled)
        CFI_STARTPROC
-       .byte 0x66,0x66,0x90    /* 5 byte nop for replacement jump */
-       .byte 0x66,0x90
-1:
-       .section .altinstr_replacement,"ax"
-2:     .byte 0xe9                   /* near jump with 32bit immediate */
-       .long copy_user_generic_c-1b /* offset */
-       .previous
-       .section .altinstructions,"a"
-       .align 8
-       .quad  copy_user_generic
-       .quad  2b
-       .byte  X86_FEATURE_REP_GOOD
-       .byte  5
-       .byte  5
-       .previous
-.Lcug:
        pushq %rbx
        CFI_ADJUST_CFA_OFFSET 8
        CFI_REL_OFFSET rbx, 0
+       pushq %rcx
+       CFI_ADJUST_CFA_OFFSET 8
+       CFI_REL_OFFSET rcx, 0
        xorl %eax,%eax          /*zero for the exception handler */
 
 #ifdef FIX_ALIGNMENT
@@ -179,6 +184,9 @@ ENTRY(copy_user_generic)
 
        CFI_REMEMBER_STATE
 .Lende:
+       popq %rcx
+       CFI_ADJUST_CFA_OFFSET -8
+       CFI_RESTORE rcx
        popq %rbx
        CFI_ADJUST_CFA_OFFSET -8
        CFI_RESTORE rbx
@@ -265,6 +273,8 @@ ENTRY(copy_user_generic)
        addl %ecx,%edx
        /* edx: bytes to zero, rdi: dest, eax:zero */
 .Lzero_rest:
+       cmpl $0,(%rsp)
+       jz   .Le_zero
        movq %rdx,%rcx
 .Le_byte:
        xorl %eax,%eax
@@ -286,6 +296,7 @@ ENDPROC(copy_user_generic)
  /* rdi        destination
   * rsi source
   * rdx count
+  * ecx zero flag
   *
   * Output:
   * eax uncopied bytes or 0 if successfull.
@@ -296,25 +307,48 @@ ENDPROC(copy_user_generic)
   * And more would be dangerous because both Intel and AMD have
   * errata with rep movsq > 4GB. If someone feels the need to fix
   * this please consider this.
-   */
-copy_user_generic_c:
+  */
+ENTRY(copy_user_generic_string)
        CFI_STARTPROC
+       movl %ecx,%r8d          /* save zero flag */
        movl %edx,%ecx
        shrl $3,%ecx
        andl $7,%edx    
+       jz   10f
 1:     rep 
        movsq 
        movl %edx,%ecx
 2:     rep
        movsb
-4:     movl %ecx,%eax
+9:     movl %ecx,%eax
        ret
-3:     lea (%rdx,%rcx,8),%rax
+
+       /* multiple of 8 byte */
+10:    rep
+       movsq
+       xor %eax,%eax
        ret
+
+       /* exception handling */
+3:      lea (%rdx,%rcx,8),%rax /* exception on quad loop */
+       jmp 6f
+5:     movl %ecx,%eax          /* exception on byte loop */
+       /* eax: left over bytes */
+6:     testl %r8d,%r8d         /* zero flag set? */
+       jz 7f
+       movl %eax,%ecx          /* initialize x86 loop counter */
+       push %rax
+       xorl %eax,%eax
+8:     rep
+       stosb                   /* zero the rest */
+11:    pop %rax
+7:     ret
        CFI_ENDPROC
 END(copy_user_generic_c)
 
        .section __ex_table,"a"
        .quad 1b,3b
-       .quad 2b,4b
+       .quad 2b,5b
+       .quad 8b,11b
+       .quad 10b,3b
        .previous

diff --git a/include/asm-x86_64/uaccess.h b/include/asm-x86_64/uaccess.h
index 1e1fa003daa3364ca9a2e96c0c1e7af75216cbd5..bc6812009bd73cdd78c3794f79d4f1954a0c2067 100644 (file)
--- a/include/asm-x86_64/uaccess.h
+++ b/include/asm-x86_64/uaccess.h
@@ -238,6 +238,7 @@ do {                                                                        \
 
 /* Handles exceptions in both to and from, but doesn't do access_ok */
 extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); 
+extern unsigned long copy_user_generic_dontzero(void *to, const void *from, unsigned len);
 
 extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); 
 extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len); 
@@ -303,7 +304,6 @@ static __always_inline int __copy_to_user(void __user *dst, const void *src, uns
        }
 }      
 
-
 static __always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
 { 
        int ret = 0;
@@ -352,7 +352,7 @@ long strlen_user(const char __user *str);
 unsigned long clear_user(void __user *mem, unsigned long len);
 unsigned long __clear_user(void __user *mem, unsigned long len);
 
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+extern long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size);
+#define __copy_to_user_inatomic copy_user_generic
 
 #endif /* __X86_64_UACCESS_H */