Commit 2ba64741 authored by Dominik Brodowski's avatar Dominik Brodowski Committed by Ingo Molnar

x86/entry/64: Move ENTER_IRQ_STACK from interrupt macro to interrupt_entry

Moving the switch to IRQ stack from the interrupt macro to the helper
function requires some trickery: All ENTER_IRQ_STACK really cares about
is where the "original" stack -- meaning the GP registers etc. -- is
stored. Therefore, we need to offset the stored RSP value by 8 whenever
ENTER_IRQ_STACK is called from within a function. In such cases, and
after switching to the IRQ stack, we need to push the "original" return
address (i.e. the return address from the call to the interrupt entry
function) to the IRQ stack.

This trickery allows us to carve another .85k from the text size (it
would be more except for the additional unwind hints):

   text	   data	    bss	    dec	    hex	filename
  18006	      0	      0	  18006	   4656	entry_64.o-orig
  17158	      0	      0	  17158	   4306	entry_64.o
Signed-off-by: default avatarDominik Brodowski <linux@dominikbrodowski.net>
Acked-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: David Woodhouse <dwmw2@infradead.org>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: dan.j.williams@intel.com
Link: http://lkml.kernel.org/r/20180220210113.6725-3-linux@dominikbrodowski.netSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 0e34d226
...@@ -448,9 +448,19 @@ END(irq_entries_start) ...@@ -448,9 +448,19 @@ END(irq_entries_start)
* *
* The invariant is that, if irq_count != -1, then the IRQ stack is in use. * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
*/ */
.macro ENTER_IRQ_STACK regs=1 old_rsp .macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0
DEBUG_ENTRY_ASSERT_IRQS_OFF DEBUG_ENTRY_ASSERT_IRQS_OFF
.if \save_ret
/*
* If save_ret is set, the original stack contains one additional
* entry -- the return address. Therefore, move the address one
* entry below %rsp to \old_rsp.
*/
leaq 8(%rsp), \old_rsp
.else
movq %rsp, \old_rsp movq %rsp, \old_rsp
.endif
.if \regs .if \regs
UNWIND_HINT_REGS base=\old_rsp UNWIND_HINT_REGS base=\old_rsp
...@@ -496,6 +506,15 @@ END(irq_entries_start) ...@@ -496,6 +506,15 @@ END(irq_entries_start)
.if \regs .if \regs
UNWIND_HINT_REGS indirect=1 UNWIND_HINT_REGS indirect=1
.endif .endif
.if \save_ret
/*
* Push the return address to the stack. This return address can
* be found at the "real" original RSP, which was offset by 8 at
* the beginning of this macro.
*/
pushq -8(\old_rsp)
.endif
.endm .endm
/* /*
...@@ -531,22 +550,7 @@ ENTRY(interrupt_entry) ...@@ -531,22 +550,7 @@ ENTRY(interrupt_entry)
PUSH_AND_CLEAR_REGS save_ret=1 PUSH_AND_CLEAR_REGS save_ret=1
ENCODE_FRAME_POINTER 8 ENCODE_FRAME_POINTER 8
ret testb $3, CS+8(%rsp)
END(interrupt_entry)
/* 0(%rsp): ~(interrupt number) */
.macro interrupt func
cld
testb $3, CS-ORIG_RAX(%rsp)
jz 1f
SWAPGS
call switch_to_thread_stack
1:
call interrupt_entry
testb $3, CS(%rsp)
jz 1f jz 1f
/* /*
...@@ -564,10 +568,26 @@ END(interrupt_entry) ...@@ -564,10 +568,26 @@ END(interrupt_entry)
CALL_enter_from_user_mode CALL_enter_from_user_mode
1: 1:
ENTER_IRQ_STACK old_rsp=%rdi ENTER_IRQ_STACK old_rsp=%rdi save_ret=1
/* We entered an interrupt context - irqs are off: */ /* We entered an interrupt context - irqs are off: */
TRACE_IRQS_OFF TRACE_IRQS_OFF
ret
END(interrupt_entry)
/* 0(%rsp): ~(interrupt number) */
.macro interrupt func
cld
testb $3, CS-ORIG_RAX(%rsp)
jz 1f
SWAPGS
call switch_to_thread_stack
1:
call interrupt_entry
UNWIND_HINT_REGS indirect=1
call \func /* rdi points to pt_regs */ call \func /* rdi points to pt_regs */
.endm .endm
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment