[PATCH] x86_64: Clean up copy_*_user

- Remove optimization for old B stepping Opteron
- Make the fast path for copies with a multiple of eight length faster.
- Minor instruction rearrangement to hopefully avoid a pipeline
stall or two.
- Add comment about errata to consider.

Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/arch/x86_64/lib/copy_user.S b/arch/x86_64/lib/copy_user.S
index dfa358b05c893e17127178fcd8504ebfc0ce556d..79422b6559c33008dfb485ebf1638d5dfd71f973 100644 (file)
--- a/arch/x86_64/lib/copy_user.S
+++ b/arch/x86_64/lib/copy_user.S
@@ -4,12 +4,9 @@
  * Functions to copy from and to user space.           
  */             
 
-#define FIX_ALIGNMENT 1
-               
        #include <asm/current.h>
        #include <asm/asm-offsets.h>
        #include <asm/thread_info.h>
-       #include <asm/cpufeature.h>
 
 /* Standard copy_to_user with segment limit checking */                
        .globl copy_to_user
@@ -21,23 +18,7 @@ copy_to_user:
        jc  bad_to_user
        cmpq threadinfo_addr_limit(%rax),%rcx
        jae bad_to_user
-2:     
-       .byte 0xe9      /* 32bit jump */
-       .long .Lcug-1f
-1:
-
-       .section .altinstr_replacement,"ax"
-3:     .byte 0xe9                      /* replacement jmp with 8 bit immediate */
-       .long copy_user_generic_c-1b    /* offset */
-       .previous
-       .section .altinstructions,"a"
-       .align 8
-       .quad  2b
-       .quad  3b
-       .byte  X86_FEATURE_K8_C
-       .byte  5
-       .byte  5
-       .previous
+       jmp copy_user_generic
 
 /* Standard copy_from_user with segment limit checking */      
        .globl copy_from_user
@@ -72,223 +53,44 @@ bad_to_user:
  * rsi source
  * rdx count
  *
+ * Only 4GB of copy is supported. This shouldn't be a problem
+ * because the kernel normally only writes from/to page sized chunks
+ * even if user space passed a longer buffer.
+ * And more would be dangerous because both Intel and AMD have
+ * errata with rep movsq > 4GB. If someone feels the need to fix
+ * this please consider this.
+ *
  * Output:             
  * eax uncopied bytes or 0 if successful.
  */
-       .globl copy_user_generic        
-       .p2align 4
-copy_user_generic:     
-       .byte 0x66,0x66,0x90    /* 5 byte nop for replacement jump */   
-       .byte 0x66,0x90
-1:             
-       .section .altinstr_replacement,"ax"
-2:     .byte 0xe9                   /* near jump with 32bit immediate */
-       .long copy_user_generic_c-1b /* offset */
-       .previous
-       .section .altinstructions,"a"
-       .align 8
-       .quad  copy_user_generic
-       .quad  2b
-       .byte  X86_FEATURE_K8_C
-       .byte  5
-       .byte  5
-       .previous
-.Lcug: 
-       pushq %rbx
-       xorl %eax,%eax          /*zero for the exception handler */
-
-#ifdef FIX_ALIGNMENT
-       /* check for bad alignment of destination */
-       movl %edi,%ecx
-       andl $7,%ecx
-       jnz  .Lbad_alignment
-.Lafter_bad_alignment:
-#endif
 
-       movq %rdx,%rcx
-
-       movl $64,%ebx   
-       shrq $6,%rdx
-       decq %rdx
-       js   .Lhandle_tail
-       
-       .p2align 4
-.Lloop:
-.Ls1:  movq (%rsi),%r11
-.Ls2:  movq 1*8(%rsi),%r8
-.Ls3:  movq 2*8(%rsi),%r9
-.Ls4:  movq 3*8(%rsi),%r10
-.Ld1:  movq %r11,(%rdi)
-.Ld2:  movq %r8,1*8(%rdi)
-.Ld3:  movq %r9,2*8(%rdi)
-.Ld4:  movq %r10,3*8(%rdi)
-               
-.Ls5:  movq 4*8(%rsi),%r11
-.Ls6:  movq 5*8(%rsi),%r8
-.Ls7:  movq 6*8(%rsi),%r9
-.Ls8:  movq 7*8(%rsi),%r10
-.Ld5:  movq %r11,4*8(%rdi)
-.Ld6:  movq %r8,5*8(%rdi)
-.Ld7:  movq %r9,6*8(%rdi)
-.Ld8:  movq %r10,7*8(%rdi)
-       
-       decq %rdx
-
-       leaq 64(%rsi),%rsi
-       leaq 64(%rdi),%rdi
-       
-       jns  .Lloop
-
-       .p2align 4
-.Lhandle_tail:
-       movl %ecx,%edx
-       andl $63,%ecx
-       shrl $3,%ecx
-       jz   .Lhandle_7
-       movl $8,%ebx
-       .p2align 4
-.Lloop_8:
-.Ls9:  movq (%rsi),%r8
-.Ld9:  movq %r8,(%rdi)
-       decl %ecx
-       leaq 8(%rdi),%rdi
-       leaq 8(%rsi),%rsi
-       jnz .Lloop_8
-       
-.Lhandle_7:            
-       movl %edx,%ecx  
-       andl $7,%ecx
-       jz   .Lende
-       .p2align 4
-.Lloop_1:
-.Ls10: movb (%rsi),%bl
-.Ld10: movb %bl,(%rdi)
-       incq %rdi
-       incq %rsi
-       decl %ecx
-       jnz .Lloop_1
-                       
-.Lende:
-       popq %rbx
-       ret     
-
-#ifdef FIX_ALIGNMENT                           
-       /* align destination */
-       .p2align 4
-.Lbad_alignment:
-       movl $8,%r9d
-       subl %ecx,%r9d
-       movl %r9d,%ecx
-       cmpq %r9,%rdx
-       jz   .Lhandle_7
-       js   .Lhandle_7
-.Lalign_1:             
-.Ls11: movb (%rsi),%bl
-.Ld11: movb %bl,(%rdi)
-       incq %rsi
-       incq %rdi
-       decl %ecx
-       jnz .Lalign_1
-       subq %r9,%rdx
-       jmp .Lafter_bad_alignment
-#endif
-       
-       /* table sorted by exception address */ 
-       .section __ex_table,"a"
-       .align 8
-       .quad .Ls1,.Ls1e
-       .quad .Ls2,.Ls2e
-       .quad .Ls3,.Ls3e
-       .quad .Ls4,.Ls4e        
-       .quad .Ld1,.Ls1e
-       .quad .Ld2,.Ls2e
-       .quad .Ld3,.Ls3e
-       .quad .Ld4,.Ls4e
-       .quad .Ls5,.Ls5e
-       .quad .Ls6,.Ls6e
-       .quad .Ls7,.Ls7e
-       .quad .Ls8,.Ls8e        
-       .quad .Ld5,.Ls5e
-       .quad .Ld6,.Ls6e
-       .quad .Ld7,.Ls7e
-       .quad .Ld8,.Ls8e
-       .quad .Ls9,.Le_quad
-       .quad .Ld9,.Le_quad
-       .quad .Ls10,.Le_byte
-       .quad .Ld10,.Le_byte
-#ifdef FIX_ALIGNMENT   
-       .quad .Ls11,.Lzero_rest
-       .quad .Ld11,.Lzero_rest
-#endif
-       .quad .Le5,.Le_zero
-       .previous
-
-       /* compute 64-offset for main loop. 8 bytes accuracy with error on the 
-          pessimistic side. this is gross. it would be better to fix the 
-          interface. */        
-       /* eax: zero, ebx: 64 */
-.Ls1e:         addl $8,%eax
-.Ls2e:         addl $8,%eax
-.Ls3e:         addl $8,%eax
-.Ls4e:         addl $8,%eax
-.Ls5e:         addl $8,%eax
-.Ls6e:         addl $8,%eax
-.Ls7e:         addl $8,%eax
-.Ls8e:         addl $8,%eax
-       addq %rbx,%rdi  /* +64 */
-       subq %rax,%rdi  /* correct destination with computed offset */
-
-       shlq $6,%rdx    /* loop counter * 64 (stride length) */
-       addq %rax,%rdx  /* add offset to loopcnt */
-       andl $63,%ecx   /* remaining bytes */
-       addq %rcx,%rdx  /* add them */
-       jmp .Lzero_rest
-
-       /* exception on quad word loop in tail handling */
-       /* ecx: loopcnt/8, %edx: length, rdi: correct */
-.Le_quad:
-       shll $3,%ecx
-       andl $7,%edx
-       addl %ecx,%edx
-       /* edx: bytes to zero, rdi: dest, eax:zero */
-.Lzero_rest:
-       movq %rdx,%rcx
-.Le_byte:
-       xorl %eax,%eax
-.Le5:  rep 
-       stosb
-       /* when there is another exception while zeroing the rest just return */
-.Le_zero:              
-       movq %rdx,%rax
-       jmp .Lende
-
-       /* C stepping K8 run faster using the string copy instructions.
-          This is also a lot simpler. Use them when possible.
-          Patch in jmps to this code instead of copying it fully
-          to avoid unwanted aliasing in the exception tables. */
-               
- /* rdi        destination
-  * rsi source
-  * rdx count
-  *
-  * Output:            
-  * eax uncopied bytes or 0 if successfull.
-  */                   
-copy_user_generic_c:
+       .globl copy_user_generic
+copy_user_generic:
        movl %edx,%ecx
        shrl $3,%ecx
        andl $7,%edx    
+       jz   5f
 1:     rep 
        movsq 
        movl %edx,%ecx
+       xor %eax,%eax
 2:     rep
        movsb
-4:     movl %ecx,%eax
        ret
+       /* align here? */
+5:     xorl %eax,%eax
+6:     rep movsq
+       ret
+
+       .section .fixup,"ax"
 3:     lea (%rdx,%rcx,8),%rax
        ret
-       
+4:     movl %ecx,%eax
+       ret
+       .previous
+
        .section __ex_table,"a"
        .quad 1b,3b
        .quad 2b,4b
+       .quad 6b,4b
        .previous