x86: fixup more paravirt fallout

Use a common irq_return entry point for all the iret places, which
need the paravirt INTERRUPT return wrapper.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index be5c31d048847e0ba2ae2f76eb5a136e9ee96cb7..824e21b80aadf8c3ff35f30424265f1e87879c35 100644 (file)
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -409,7 +409,8 @@ restore_nocheck_notrace:
        RESTORE_REGS
        addl $4, %esp                   # skip orig_eax/error_code
        CFI_ADJUST_CFA_OFFSET -4
-1:     INTERRUPT_RETURN
+ENTRY(irq_return)
+       INTERRUPT_RETURN
 .section .fixup,"ax"
 iret_exc:
        pushl $0                        # no error code
@@ -418,7 +419,7 @@ iret_exc:
 .previous
 .section __ex_table,"a"
        .align 4
-       .long 1b,iret_exc
+       .long irq_return,iret_exc
 .previous
 
        CFI_RESTORE_STATE
@@ -865,20 +866,16 @@ nmi_espfix_stack:
        RESTORE_REGS
        lss 12+4(%esp), %esp            # back to espfix stack
        CFI_ADJUST_CFA_OFFSET -24
-1:     INTERRUPT_RETURN
+       jmp irq_return
        CFI_ENDPROC
-.section __ex_table,"a"
-       .align 4
-       .long 1b,iret_exc
-.previous
 KPROBE_END(nmi)
 
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
-1:     iret
+       iret
 .section __ex_table,"a"
        .align 4
-       .long 1b,iret_exc
+       .long native_iret, iret_exc
 .previous
 END(native_iret)
 

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index c7341e81941cc0e7d6863e317953316c2f045392..6be39a387c5ae66780b0b5f946a71c336980fea8 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -581,16 +581,24 @@ retint_restore_args:      /* return to kernel space */
         */
        TRACE_IRQS_IRETQ
 restore_args:
-       RESTORE_ARGS 0,8,0                                              
-#ifdef CONFIG_PARAVIRT
+       RESTORE_ARGS 0,8,0
+
+ENTRY(irq_return)
        INTERRUPT_RETURN
-#endif
+
+       .section __ex_table, "a"
+       .quad irq_return, bad_iret
+       .previous
+
+#ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
        iretq
 
        .section __ex_table,"a"
        .quad native_iret, bad_iret
        .previous
+#endif
+
        .section .fixup,"ax"
 bad_iret:
        /*
@@ -804,7 +812,7 @@ paranoid_swapgs\trace:
        SWAPGS_UNSAFE_STACK
 paranoid_restore\trace:
        RESTORE_ALL 8
-       INTERRUPT_RETURN
+       jmp irq_return
 paranoid_userspace\trace:
        GET_THREAD_INFO(%rcx)
        movl threadinfo_flags(%rcx),%ebx
@@ -919,7 +927,7 @@ error_kernelspace:
           iret run with kernel gs again, so don't set the user space flag.
           B stepping K8s sometimes report an truncated RIP for IRET 
           exceptions returning to compat mode. Check for these here too. */
-       leaq native_iret(%rip),%rbp
+       leaq irq_return(%rip),%rbp
        cmpq %rbp,RIP(%rsp) 
        je   error_swapgs
        movl %ebp,%ebp  /* zero extend */