Commit 625dbc3b authored by Frederic Weisbecker's avatar Frederic Weisbecker

x86: Save rbp in pt_regs on irq entry

From the x86_64 low level interrupt handlers, the frame pointer is
saved right after the partial pt_regs frame.

rbp is not supposed to be part of the irq partial saved registers,
but it only requires to extend the pt_regs frame by 8 bytes to
do so, plus a tiny stack offset fixup on irq exit.

This changes a bit the semantics or get_irq_entry() that is supposed
to provide only the value of caller saved registers and the cpu
saved frame. However it's a win for unwinders that can walk through
stack frames on top of get_irq_regs() snapshots.

A noticeable impact is that it makes perf events cpu-clock and
task-clock events based callchains working on x86_64.

Let's then save rbp into the irq pt_regs.

As a result with:

	perf record -e cpu-clock perf bench sched messaging
	perf report --stdio

Before:
    20.94%             perf  [kernel.kallsyms]        [k] lock_acquire
                       |
                       --- lock_acquire
                          |
                          |--44.01%-- __write_nocancel
                          |
                          |--43.18%-- __read
                          |
                          |--6.08%-- fork
                          |          create_worker
                          |
                          |--0.88%-- _dl_fixup
                          |
                          |--0.65%-- do_lookup_x
                          |
                          |--0.53%-- __GI___libc_read
                           --4.67%-- [...]

After:
    19.23%         perf  [kernel.kallsyms]    [k] __lock_acquire
                   |
                   --- __lock_acquire
                      |
                      |--97.74%-- lock_acquire
                      |          |
                      |          |--21.82%-- _raw_spin_lock
                      |          |          |
                      |          |          |--37.26%-- unix_stream_recvmsg
                      |          |          |          sock_aio_read
                      |          |          |          do_sync_read
                      |          |          |          vfs_read
                      |          |          |          sys_read
                      |          |          |          system_call
                      |          |          |          __read
                      |          |          |
                      |          |          |--24.09%-- unix_stream_sendmsg
                      |          |          |          sock_aio_write
                      |          |          |          do_sync_write
                      |          |          |          vfs_write
                      |          |          |          sys_write
                      |          |          |          system_call
                      |          |          |          __write_nocancel

v2: Fix cfi annotations.
Reported-by: default avatarSoeren Sandmann Pedersen <sandmann@redhat.com>
Signed-off-by: default avatarFrederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: H. Peter Anvin <hpa@zytor.com
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Stephane Eranian <eranian@google.com>
Cc: Jan Beulich <JBeulich@novell.com>
parent 39a6eebd
...@@ -299,17 +299,21 @@ ENDPROC(native_usergs_sysret64) ...@@ -299,17 +299,21 @@ ENDPROC(native_usergs_sysret64)
ENTRY(save_args) ENTRY(save_args)
XCPT_FRAME XCPT_FRAME
cld cld
movq_cfi rdi, RDI+16-ARGOFFSET /*
movq_cfi rsi, RSI+16-ARGOFFSET * start from rbp in pt_regs and jump over
movq_cfi rdx, RDX+16-ARGOFFSET * return address.
movq_cfi rcx, RCX+16-ARGOFFSET */
movq_cfi rax, RAX+16-ARGOFFSET movq_cfi rdi, RDI+8-RBP
movq_cfi r8, R8+16-ARGOFFSET movq_cfi rsi, RSI+8-RBP
movq_cfi r9, R9+16-ARGOFFSET movq_cfi rdx, RDX+8-RBP
movq_cfi r10, R10+16-ARGOFFSET movq_cfi rcx, RCX+8-RBP
movq_cfi r11, R11+16-ARGOFFSET movq_cfi rax, RAX+8-RBP
movq_cfi r8, R8+8-RBP
leaq -ARGOFFSET+16(%rsp),%rdi /* arg1 for handler */ movq_cfi r9, R9+8-RBP
movq_cfi r10, R10+8-RBP
movq_cfi r11, R11+8-RBP
leaq -RBP+8(%rsp),%rdi /* arg1 for handler */
movq_cfi rbp, 8 /* push %rbp */ movq_cfi rbp, 8 /* push %rbp */
leaq 8(%rsp), %rbp /* mov %rsp, %ebp */ leaq 8(%rsp), %rbp /* mov %rsp, %ebp */
testl $3, CS(%rdi) testl $3, CS(%rdi)
...@@ -782,8 +786,9 @@ END(interrupt) ...@@ -782,8 +786,9 @@ END(interrupt)
/* 0(%rsp): ~(interrupt number) */ /* 0(%rsp): ~(interrupt number) */
.macro interrupt func .macro interrupt func
subq $ORIG_RAX-ARGOFFSET+8, %rsp /* reserve pt_regs for scratch regs and rbp */
CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+8 subq $ORIG_RAX-RBP, %rsp
CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
call save_args call save_args
PARTIAL_FRAME 0 PARTIAL_FRAME 0
call \func call \func
...@@ -808,9 +813,14 @@ ret_from_intr: ...@@ -808,9 +813,14 @@ ret_from_intr:
TRACE_IRQS_OFF TRACE_IRQS_OFF
decl PER_CPU_VAR(irq_count) decl PER_CPU_VAR(irq_count)
leaveq leaveq
CFI_RESTORE rbp CFI_RESTORE rbp
CFI_DEF_CFA_REGISTER rsp CFI_DEF_CFA_REGISTER rsp
CFI_ADJUST_CFA_OFFSET -8 CFI_ADJUST_CFA_OFFSET -8
/* we did not save rbx, restore only from ARGOFFSET */
addq $8, %rsp
CFI_ADJUST_CFA_OFFSET -8
exit_intr: exit_intr:
GET_THREAD_INFO(%rcx) GET_THREAD_INFO(%rcx)
testl $3,CS-ARGOFFSET(%rsp) testl $3,CS-ARGOFFSET(%rsp)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment