Commit f24d6f26 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 asm updates from Thomas Gleixner:
 "The lowlevel and ASM code updates for x86:

   - Make stack trace unwinding more reliable

   - ASM instruction updates for better code generation

   - Various cleanups"

* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/entry/64: Add two more instruction suffixes
  x86/asm/64: Use 32-bit XOR to zero registers
  x86/build/vdso: Simplify 'cmd_vdso2c'
  x86/build/vdso: Remove unused vdso-syms.lds
  x86/stacktrace: Enable HAVE_RELIABLE_STACKTRACE for the ORC unwinder
  x86/unwind/orc: Detect the end of the stack
  x86/stacktrace: Do not fail for ORC with regs on stack
  x86/stacktrace: Clarify the reliable success paths
  x86/stacktrace: Remove STACKTRACE_DUMP_ONCE
  x86/stacktrace: Do not unwind after user regs
  x86/asm: Use CC_SET/CC_OUT in percpu_cmpxchg8b_double() to micro-optimize code generation
parents b9b8e5b7 6709812f
...@@ -180,7 +180,7 @@ config X86 ...@@ -180,7 +180,7 @@ config X86
select HAVE_PERF_USER_STACK_DUMP select HAVE_PERF_USER_STACK_DUMP
select HAVE_RCU_TABLE_FREE select HAVE_RCU_TABLE_FREE
select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE if X86_64 && UNWINDER_FRAME_POINTER && STACK_VALIDATION select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR
select HAVE_STACK_VALIDATION if X86_64 select HAVE_STACK_VALIDATION if X86_64
select HAVE_RSEQ select HAVE_RSEQ
......
...@@ -75,7 +75,7 @@ ...@@ -75,7 +75,7 @@
* %r9 * %r9
*/ */
__load_partial: __load_partial:
xor %r9, %r9 xor %r9d, %r9d
pxor MSG, MSG pxor MSG, MSG
mov LEN, %r8 mov LEN, %r8
......
...@@ -66,7 +66,7 @@ ...@@ -66,7 +66,7 @@
* %r9 * %r9
*/ */
__load_partial: __load_partial:
xor %r9, %r9 xor %r9d, %r9d
pxor MSG0, MSG0 pxor MSG0, MSG0
pxor MSG1, MSG1 pxor MSG1, MSG1
......
...@@ -59,7 +59,7 @@ ...@@ -59,7 +59,7 @@
* %r9 * %r9
*/ */
__load_partial: __load_partial:
xor %r9, %r9 xor %r9d, %r9d
pxor MSG, MSG pxor MSG, MSG
mov LEN, %r8 mov LEN, %r8
......
...@@ -258,7 +258,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff ...@@ -258,7 +258,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff
.macro GCM_INIT Iv SUBKEY AAD AADLEN .macro GCM_INIT Iv SUBKEY AAD AADLEN
mov \AADLEN, %r11 mov \AADLEN, %r11
mov %r11, AadLen(%arg2) # ctx_data.aad_length = aad_length mov %r11, AadLen(%arg2) # ctx_data.aad_length = aad_length
xor %r11, %r11 xor %r11d, %r11d
mov %r11, InLen(%arg2) # ctx_data.in_length = 0 mov %r11, InLen(%arg2) # ctx_data.in_length = 0
mov %r11, PBlockLen(%arg2) # ctx_data.partial_block_length = 0 mov %r11, PBlockLen(%arg2) # ctx_data.partial_block_length = 0
mov %r11, PBlockEncKey(%arg2) # ctx_data.partial_block_enc_key = 0 mov %r11, PBlockEncKey(%arg2) # ctx_data.partial_block_enc_key = 0
...@@ -286,7 +286,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff ...@@ -286,7 +286,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff
movdqu HashKey(%arg2), %xmm13 movdqu HashKey(%arg2), %xmm13
add %arg5, InLen(%arg2) add %arg5, InLen(%arg2)
xor %r11, %r11 # initialise the data pointer offset as zero xor %r11d, %r11d # initialise the data pointer offset as zero
PARTIAL_BLOCK %arg3 %arg4 %arg5 %r11 %xmm8 \operation PARTIAL_BLOCK %arg3 %arg4 %arg5 %r11 %xmm8 \operation
sub %r11, %arg5 # sub partial block data used sub %r11, %arg5 # sub partial block data used
...@@ -702,7 +702,7 @@ _no_extra_mask_1_\@: ...@@ -702,7 +702,7 @@ _no_extra_mask_1_\@:
# GHASH computation for the last <16 Byte block # GHASH computation for the last <16 Byte block
GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
xor %rax,%rax xor %eax, %eax
mov %rax, PBlockLen(%arg2) mov %rax, PBlockLen(%arg2)
jmp _dec_done_\@ jmp _dec_done_\@
...@@ -737,7 +737,7 @@ _no_extra_mask_2_\@: ...@@ -737,7 +737,7 @@ _no_extra_mask_2_\@:
# GHASH computation for the last <16 Byte block # GHASH computation for the last <16 Byte block
GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
xor %rax,%rax xor %eax, %eax
mov %rax, PBlockLen(%arg2) mov %rax, PBlockLen(%arg2)
jmp _encode_done_\@ jmp _encode_done_\@
......
...@@ -463,7 +463,7 @@ _get_AAD_rest_final\@: ...@@ -463,7 +463,7 @@ _get_AAD_rest_final\@:
_get_AAD_done\@: _get_AAD_done\@:
# initialize the data pointer offset as zero # initialize the data pointer offset as zero
xor %r11, %r11 xor %r11d, %r11d
# start AES for num_initial_blocks blocks # start AES for num_initial_blocks blocks
mov arg5, %rax # rax = *Y0 mov arg5, %rax # rax = *Y0
...@@ -1770,7 +1770,7 @@ _get_AAD_rest_final\@: ...@@ -1770,7 +1770,7 @@ _get_AAD_rest_final\@:
_get_AAD_done\@: _get_AAD_done\@:
# initialize the data pointer offset as zero # initialize the data pointer offset as zero
xor %r11, %r11 xor %r11d, %r11d
# start AES for num_initial_blocks blocks # start AES for num_initial_blocks blocks
mov arg5, %rax # rax = *Y0 mov arg5, %rax # rax = *Y0
......
...@@ -113,7 +113,7 @@ ENDPROC(__morus1280_update_zero) ...@@ -113,7 +113,7 @@ ENDPROC(__morus1280_update_zero)
* %r9 * %r9
*/ */
__load_partial: __load_partial:
xor %r9, %r9 xor %r9d, %r9d
vpxor MSG, MSG, MSG vpxor MSG, MSG, MSG
mov %rcx, %r8 mov %rcx, %r8
......
...@@ -235,7 +235,7 @@ ENDPROC(__morus1280_update_zero) ...@@ -235,7 +235,7 @@ ENDPROC(__morus1280_update_zero)
* %r9 * %r9
*/ */
__load_partial: __load_partial:
xor %r9, %r9 xor %r9d, %r9d
pxor MSG_LO, MSG_LO pxor MSG_LO, MSG_LO
pxor MSG_HI, MSG_HI pxor MSG_HI, MSG_HI
......
...@@ -113,7 +113,7 @@ ENDPROC(__morus640_update_zero) ...@@ -113,7 +113,7 @@ ENDPROC(__morus640_update_zero)
* %r9 * %r9
*/ */
__load_partial: __load_partial:
xor %r9, %r9 xor %r9d, %r9d
pxor MSG, MSG pxor MSG, MSG
mov %rcx, %r8 mov %rcx, %r8
......
...@@ -96,7 +96,7 @@ ...@@ -96,7 +96,7 @@
# cleanup workspace # cleanup workspace
mov $8, %ecx mov $8, %ecx
mov %rsp, %rdi mov %rsp, %rdi
xor %rax, %rax xor %eax, %eax
rep stosq rep stosq
mov %rbp, %rsp # deallocate workspace mov %rbp, %rsp # deallocate workspace
......
...@@ -92,7 +92,7 @@ END(native_usergs_sysret64) ...@@ -92,7 +92,7 @@ END(native_usergs_sysret64)
.endm .endm
.macro TRACE_IRQS_IRETQ_DEBUG .macro TRACE_IRQS_IRETQ_DEBUG
bt $9, EFLAGS(%rsp) /* interrupts off? */ btl $9, EFLAGS(%rsp) /* interrupts off? */
jnc 1f jnc 1f
TRACE_IRQS_ON_DEBUG TRACE_IRQS_ON_DEBUG
1: 1:
...@@ -408,6 +408,7 @@ ENTRY(ret_from_fork) ...@@ -408,6 +408,7 @@ ENTRY(ret_from_fork)
1: 1:
/* kernel thread */ /* kernel thread */
UNWIND_HINT_EMPTY
movq %r12, %rdi movq %r12, %rdi
CALL_NOSPEC %rbx CALL_NOSPEC %rbx
/* /*
...@@ -701,7 +702,7 @@ retint_kernel: ...@@ -701,7 +702,7 @@ retint_kernel:
#ifdef CONFIG_PREEMPT #ifdef CONFIG_PREEMPT
/* Interrupts are off */ /* Interrupts are off */
/* Check if we need preemption */ /* Check if we need preemption */
bt $9, EFLAGS(%rsp) /* were interrupts off? */ btl $9, EFLAGS(%rsp) /* were interrupts off? */
jnc 1f jnc 1f
0: cmpl $0, PER_CPU_VAR(__preempt_count) 0: cmpl $0, PER_CPU_VAR(__preempt_count)
jnz 1f jnz 1f
......
...@@ -58,9 +58,7 @@ HOST_EXTRACFLAGS += -I$(srctree)/tools/include -I$(srctree)/include/uapi -I$(src ...@@ -58,9 +58,7 @@ HOST_EXTRACFLAGS += -I$(srctree)/tools/include -I$(srctree)/include/uapi -I$(src
hostprogs-y += vdso2c hostprogs-y += vdso2c
quiet_cmd_vdso2c = VDSO2C $@ quiet_cmd_vdso2c = VDSO2C $@
define cmd_vdso2c cmd_vdso2c = $(obj)/vdso2c $< $(<:%.dbg=%) $@
$(obj)/vdso2c $< $(<:%.dbg=%) $@
endef
$(obj)/vdso-image-%.c: $(obj)/vdso%.so.dbg $(obj)/vdso%.so $(obj)/vdso2c FORCE $(obj)/vdso-image-%.c: $(obj)/vdso%.so.dbg $(obj)/vdso%.so $(obj)/vdso2c FORCE
$(call if_changed,vdso2c) $(call if_changed,vdso2c)
......
...@@ -88,6 +88,7 @@ struct orc_entry { ...@@ -88,6 +88,7 @@ struct orc_entry {
unsigned sp_reg:4; unsigned sp_reg:4;
unsigned bp_reg:4; unsigned bp_reg:4;
unsigned type:2; unsigned type:2;
unsigned end:1;
} __packed; } __packed;
/* /*
...@@ -101,6 +102,7 @@ struct unwind_hint { ...@@ -101,6 +102,7 @@ struct unwind_hint {
s16 sp_offset; s16 sp_offset;
u8 sp_reg; u8 sp_reg;
u8 type; u8 type;
u8 end;
}; };
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -450,9 +450,10 @@ do { \ ...@@ -450,9 +450,10 @@ do { \
bool __ret; \ bool __ret; \
typeof(pcp1) __o1 = (o1), __n1 = (n1); \ typeof(pcp1) __o1 = (o1), __n1 = (n1); \
typeof(pcp2) __o2 = (o2), __n2 = (n2); \ typeof(pcp2) __o2 = (o2), __n2 = (n2); \
asm volatile("cmpxchg8b "__percpu_arg(1)"\n\tsetz %0\n\t" \ asm volatile("cmpxchg8b "__percpu_arg(1) \
: "=a" (__ret), "+m" (pcp1), "+m" (pcp2), "+d" (__o2) \ CC_SET(z) \
: "b" (__n1), "c" (__n2), "a" (__o1)); \ : CC_OUT(z) (__ret), "+m" (pcp1), "+m" (pcp2), "+a" (__o1), "+d" (__o2) \
: "b" (__n1), "c" (__n2)); \
__ret; \ __ret; \
}) })
......
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
* the debuginfo as necessary. It will also warn if it sees any * the debuginfo as necessary. It will also warn if it sees any
* inconsistencies. * inconsistencies.
*/ */
.macro UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=0 type=ORC_TYPE_CALL .macro UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=0 type=ORC_TYPE_CALL end=0
#ifdef CONFIG_STACK_VALIDATION #ifdef CONFIG_STACK_VALIDATION
.Lunwind_hint_ip_\@: .Lunwind_hint_ip_\@:
.pushsection .discard.unwind_hints .pushsection .discard.unwind_hints
...@@ -35,12 +35,14 @@ ...@@ -35,12 +35,14 @@
.short \sp_offset .short \sp_offset
.byte \sp_reg .byte \sp_reg
.byte \type .byte \type
.byte \end
.balign 4
.popsection .popsection
#endif #endif
.endm .endm
.macro UNWIND_HINT_EMPTY .macro UNWIND_HINT_EMPTY
UNWIND_HINT sp_reg=ORC_REG_UNDEFINED UNWIND_HINT sp_reg=ORC_REG_UNDEFINED end=1
.endm .endm
.macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 iret=0 .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 iret=0
...@@ -86,7 +88,7 @@ ...@@ -86,7 +88,7 @@
#else /* !__ASSEMBLY__ */ #else /* !__ASSEMBLY__ */
#define UNWIND_HINT(sp_reg, sp_offset, type) \ #define UNWIND_HINT(sp_reg, sp_offset, type, end) \
"987: \n\t" \ "987: \n\t" \
".pushsection .discard.unwind_hints\n\t" \ ".pushsection .discard.unwind_hints\n\t" \
/* struct unwind_hint */ \ /* struct unwind_hint */ \
...@@ -94,11 +96,13 @@ ...@@ -94,11 +96,13 @@
".short " __stringify(sp_offset) "\n\t" \ ".short " __stringify(sp_offset) "\n\t" \
".byte " __stringify(sp_reg) "\n\t" \ ".byte " __stringify(sp_reg) "\n\t" \
".byte " __stringify(type) "\n\t" \ ".byte " __stringify(type) "\n\t" \
".byte " __stringify(end) "\n\t" \
".balign 4 \n\t" \
".popsection\n\t" ".popsection\n\t"
#define UNWIND_HINT_SAVE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_SAVE) #define UNWIND_HINT_SAVE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_SAVE, 0)
#define UNWIND_HINT_RESTORE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_RESTORE) #define UNWIND_HINT_RESTORE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_RESTORE, 0)
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -235,7 +235,7 @@ ENTRY(secondary_startup_64) ...@@ -235,7 +235,7 @@ ENTRY(secondary_startup_64)
* address given in m16:64. * address given in m16:64.
*/ */
pushq $.Lafter_lret # put return address on stack for unwinder pushq $.Lafter_lret # put return address on stack for unwinder
xorq %rbp, %rbp # clear frame pointer xorl %ebp, %ebp # clear frame pointer
movq initial_code(%rip), %rax movq initial_code(%rip), %rax
pushq $__KERNEL_CS # set correct cs pushq $__KERNEL_CS # set correct cs
pushq %rax # target address in negative space pushq %rax # target address in negative space
......
...@@ -20,7 +20,7 @@ DEF_NATIVE(, mov64, "mov %rdi, %rax"); ...@@ -20,7 +20,7 @@ DEF_NATIVE(, mov64, "mov %rdi, %rax");
#if defined(CONFIG_PARAVIRT_SPINLOCKS) #if defined(CONFIG_PARAVIRT_SPINLOCKS)
DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%rdi)"); DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%rdi)");
DEF_NATIVE(pv_lock_ops, vcpu_is_preempted, "xor %rax, %rax"); DEF_NATIVE(pv_lock_ops, vcpu_is_preempted, "xor %eax, %eax");
#endif #endif
unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len) unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len)
......
...@@ -81,16 +81,6 @@ EXPORT_SYMBOL_GPL(save_stack_trace_tsk); ...@@ -81,16 +81,6 @@ EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE #ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
#define STACKTRACE_DUMP_ONCE(task) ({ \
static bool __section(.data.unlikely) __dumped; \
\
if (!__dumped) { \
__dumped = true; \
WARN_ON(1); \
show_stack(task, NULL); \
} \
})
static int __always_inline static int __always_inline
__save_stack_trace_reliable(struct stack_trace *trace, __save_stack_trace_reliable(struct stack_trace *trace,
struct task_struct *task) struct task_struct *task)
...@@ -99,31 +89,26 @@ __save_stack_trace_reliable(struct stack_trace *trace, ...@@ -99,31 +89,26 @@ __save_stack_trace_reliable(struct stack_trace *trace,
struct pt_regs *regs; struct pt_regs *regs;
unsigned long addr; unsigned long addr;
for (unwind_start(&state, task, NULL, NULL); !unwind_done(&state); for (unwind_start(&state, task, NULL, NULL);
!unwind_done(&state) && !unwind_error(&state);
unwind_next_frame(&state)) { unwind_next_frame(&state)) {
regs = unwind_get_entry_regs(&state, NULL); regs = unwind_get_entry_regs(&state, NULL);
if (regs) { if (regs) {
/* Success path for user tasks */
if (user_mode(regs))
goto success;
/* /*
* Kernel mode registers on the stack indicate an * Kernel mode registers on the stack indicate an
* in-kernel interrupt or exception (e.g., preemption * in-kernel interrupt or exception (e.g., preemption
* or a page fault), which can make frame pointers * or a page fault), which can make frame pointers
* unreliable. * unreliable.
*/ */
if (!user_mode(regs))
return -EINVAL;
/* if (IS_ENABLED(CONFIG_FRAME_POINTER))
* The last frame contains the user mode syscall
* pt_regs. Skip it and finish the unwind.
*/
unwind_next_frame(&state);
if (!unwind_done(&state)) {
STACKTRACE_DUMP_ONCE(task);
return -EINVAL; return -EINVAL;
} }
break;
}
addr = unwind_get_return_address(&state); addr = unwind_get_return_address(&state);
...@@ -132,21 +117,22 @@ __save_stack_trace_reliable(struct stack_trace *trace, ...@@ -132,21 +117,22 @@ __save_stack_trace_reliable(struct stack_trace *trace,
* generated code which __kernel_text_address() doesn't know * generated code which __kernel_text_address() doesn't know
* about. * about.
*/ */
if (!addr) { if (!addr)
STACKTRACE_DUMP_ONCE(task);
return -EINVAL; return -EINVAL;
}
if (save_stack_address(trace, addr, false)) if (save_stack_address(trace, addr, false))
return -EINVAL; return -EINVAL;
} }
/* Check for stack corruption */ /* Check for stack corruption */
if (unwind_error(&state)) { if (unwind_error(&state))
STACKTRACE_DUMP_ONCE(task); return -EINVAL;
/* Success path for non-user tasks, i.e. kthreads and idle tasks */
if (!(task->flags & (PF_KTHREAD | PF_IDLE)))
return -EINVAL; return -EINVAL;
}
success:
if (trace->nr_entries < trace->max_entries) if (trace->nr_entries < trace->max_entries)
trace->entries[trace->nr_entries++] = ULONG_MAX; trace->entries[trace->nr_entries++] = ULONG_MAX;
......
...@@ -198,7 +198,7 @@ static int orc_sort_cmp(const void *_a, const void *_b) ...@@ -198,7 +198,7 @@ static int orc_sort_cmp(const void *_a, const void *_b)
* whitelisted .o files which didn't get objtool generation. * whitelisted .o files which didn't get objtool generation.
*/ */
orc_a = cur_orc_table + (a - cur_orc_ip_table); orc_a = cur_orc_table + (a - cur_orc_ip_table);
return orc_a->sp_reg == ORC_REG_UNDEFINED ? -1 : 1; return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
} }
#ifdef CONFIG_MODULES #ifdef CONFIG_MODULES
...@@ -352,7 +352,7 @@ static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr ...@@ -352,7 +352,7 @@ static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr
bool unwind_next_frame(struct unwind_state *state) bool unwind_next_frame(struct unwind_state *state)
{ {
unsigned long ip_p, sp, orig_ip, prev_sp = state->sp; unsigned long ip_p, sp, orig_ip = state->ip, prev_sp = state->sp;
enum stack_type prev_type = state->stack_info.type; enum stack_type prev_type = state->stack_info.type;
struct orc_entry *orc; struct orc_entry *orc;
bool indirect = false; bool indirect = false;
...@@ -363,9 +363,9 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -363,9 +363,9 @@ bool unwind_next_frame(struct unwind_state *state)
/* Don't let modules unload while we're reading their ORC data. */ /* Don't let modules unload while we're reading their ORC data. */
preempt_disable(); preempt_disable();
/* Have we reached the end? */ /* End-of-stack check for user tasks: */
if (state->regs && user_mode(state->regs)) if (state->regs && user_mode(state->regs))
goto done; goto the_end;
/* /*
* Find the orc_entry associated with the text address. * Find the orc_entry associated with the text address.
...@@ -374,9 +374,16 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -374,9 +374,16 @@ bool unwind_next_frame(struct unwind_state *state)
* calls and calls to noreturn functions. * calls and calls to noreturn functions.
*/ */
orc = orc_find(state->signal ? state->ip : state->ip - 1); orc = orc_find(state->signal ? state->ip : state->ip - 1);
if (!orc || orc->sp_reg == ORC_REG_UNDEFINED) if (!orc)
goto done; goto err;
orig_ip = state->ip;
/* End-of-stack check for kernel threads: */
if (orc->sp_reg == ORC_REG_UNDEFINED) {
if (!orc->end)
goto err;
goto the_end;
}
/* Find the previous frame's stack: */ /* Find the previous frame's stack: */
switch (orc->sp_reg) { switch (orc->sp_reg) {
...@@ -402,7 +409,7 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -402,7 +409,7 @@ bool unwind_next_frame(struct unwind_state *state)
if (!state->regs || !state->full_regs) { if (!state->regs || !state->full_regs) {
orc_warn("missing regs for base reg R10 at ip %pB\n", orc_warn("missing regs for base reg R10 at ip %pB\n",
(void *)state->ip); (void *)state->ip);
goto done; goto err;
} }
sp = state->regs->r10; sp = state->regs->r10;
break; break;
...@@ -411,7 +418,7 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -411,7 +418,7 @@ bool unwind_next_frame(struct unwind_state *state)
if (!state->regs || !state->full_regs) { if (!state->regs || !state->full_regs) {
orc_warn("missing regs for base reg R13 at ip %pB\n", orc_warn("missing regs for base reg R13 at ip %pB\n",
(void *)state->ip); (void *)state->ip);
goto done; goto err;
} }
sp = state->regs->r13; sp = state->regs->r13;
break; break;
...@@ -420,7 +427,7 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -420,7 +427,7 @@ bool unwind_next_frame(struct unwind_state *state)
if (!state->regs || !state->full_regs) { if (!state->regs || !state->full_regs) {
orc_warn("missing regs for base reg DI at ip %pB\n", orc_warn("missing regs for base reg DI at ip %pB\n",
(void *)state->ip); (void *)state->ip);
goto done; goto err;
} }
sp = state->regs->di; sp = state->regs->di;
break; break;
...@@ -429,7 +436,7 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -429,7 +436,7 @@ bool unwind_next_frame(struct unwind_state *state)
if (!state->regs || !state->full_regs) { if (!state->regs || !state->full_regs) {
orc_warn("missing regs for base reg DX at ip %pB\n", orc_warn("missing regs for base reg DX at ip %pB\n",
(void *)state->ip); (void *)state->ip);
goto done; goto err;
} }
sp = state->regs->dx; sp = state->regs->dx;
break; break;
...@@ -437,12 +444,12 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -437,12 +444,12 @@ bool unwind_next_frame(struct unwind_state *state)
default: default:
orc_warn("unknown SP base reg %d for ip %pB\n", orc_warn("unknown SP base reg %d for ip %pB\n",
orc->sp_reg, (void *)state->ip); orc->sp_reg, (void *)state->ip);
goto done; goto err;
} }
if (indirect) { if (indirect) {
if (!deref_stack_reg(state, sp, &sp)) if (!deref_stack_reg(state, sp, &sp))
goto done; goto err;
} }
/* Find IP, SP and possibly regs: */ /* Find IP, SP and possibly regs: */
...@@ -451,7 +458,7 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -451,7 +458,7 @@ bool unwind_next_frame(struct unwind_state *state)
ip_p = sp - sizeof(long); ip_p = sp - sizeof(long);
if (!deref_stack_reg(state, ip_p, &state->ip)) if (!deref_stack_reg(state, ip_p, &state->ip))
goto done; goto err;
state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx,
state->ip, (void *)ip_p); state->ip, (void *)ip_p);
...@@ -465,7 +472,7 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -465,7 +472,7 @@ bool unwind_next_frame(struct unwind_state *state)
if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) {
orc_warn("can't dereference registers at %p for ip %pB\n", orc_warn("can't dereference registers at %p for ip %pB\n",
(void *)sp, (void *)orig_ip); (void *)sp, (void *)orig_ip);
goto done; goto err;
} }
state->regs = (struct pt_regs *)sp; state->regs = (struct pt_regs *)sp;
...@@ -477,7 +484,7 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -477,7 +484,7 @@ bool unwind_next_frame(struct unwind_state *state)
if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) {
orc_warn("can't dereference iret registers at %p for ip %pB\n", orc_warn("can't dereference iret registers at %p for ip %pB\n",
(void *)sp, (void *)orig_ip); (void *)sp, (void *)orig_ip);
goto done; goto err;
} }
state->regs = (void *)sp - IRET_FRAME_OFFSET; state->regs = (void *)sp - IRET_FRAME_OFFSET;
...@@ -500,18 +507,18 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -500,18 +507,18 @@ bool unwind_next_frame(struct unwind_state *state)
case ORC_REG_PREV_SP: case ORC_REG_PREV_SP:
if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp)) if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp))
goto done; goto err;
break; break;
case ORC_REG_BP: case ORC_REG_BP:
if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp)) if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp))
goto done; goto err;
break; break;
default: default:
orc_warn("unknown BP base reg %d for ip %pB\n", orc_warn("unknown BP base reg %d for ip %pB\n",
orc->bp_reg, (void *)orig_ip); orc->bp_reg, (void *)orig_ip);
goto done; goto err;
} }
/* Prevent a recursive loop due to bad ORC data: */ /* Prevent a recursive loop due to bad ORC data: */
...@@ -520,13 +527,16 @@ bool unwind_next_frame(struct unwind_state *state) ...@@ -520,13 +527,16 @@ bool unwind_next_frame(struct unwind_state *state)
state->sp <= prev_sp) { state->sp <= prev_sp) {
orc_warn("stack going in the wrong direction? ip=%pB\n", orc_warn("stack going in the wrong direction? ip=%pB\n",
(void *)orig_ip); (void *)orig_ip);
goto done; goto err;
} }
preempt_enable(); preempt_enable();
return true; return true;
done: err:
state->error = true;
the_end:
preempt_enable(); preempt_enable();
state->stack_info.type = STACK_TYPE_UNKNOWN; state->stack_info.type = STACK_TYPE_UNKNOWN;
return false; return false;
......
...@@ -256,7 +256,7 @@ ENTRY(__memcpy_mcsafe) ...@@ -256,7 +256,7 @@ ENTRY(__memcpy_mcsafe)
/* Copy successful. Return zero */ /* Copy successful. Return zero */
.L_done_memcpy_trap: .L_done_memcpy_trap:
xorq %rax, %rax xorl %eax, %eax
ret ret
ENDPROC(__memcpy_mcsafe) ENDPROC(__memcpy_mcsafe)
EXPORT_SYMBOL_GPL(__memcpy_mcsafe) EXPORT_SYMBOL_GPL(__memcpy_mcsafe)
......
...@@ -137,7 +137,7 @@ ENTRY(restore_registers) ...@@ -137,7 +137,7 @@ ENTRY(restore_registers)
/* Saved in save_processor_state. */ /* Saved in save_processor_state. */
lgdt saved_context_gdt_desc(%rax) lgdt saved_context_gdt_desc(%rax)
xorq %rax, %rax xorl %eax, %eax
/* tell the hibernation core that we've just restored the memory */ /* tell the hibernation core that we've just restored the memory */
movq %rax, in_suspend(%rip) movq %rax, in_suspend(%rip)
......
...@@ -53,22 +53,6 @@ $(vobjs): KBUILD_CFLAGS += $(CFL) ...@@ -53,22 +53,6 @@ $(vobjs): KBUILD_CFLAGS += $(CFL)
CFLAGS_REMOVE_vdso-note.o = -pg -fprofile-arcs -ftest-coverage CFLAGS_REMOVE_vdso-note.o = -pg -fprofile-arcs -ftest-coverage
CFLAGS_REMOVE_um_vdso.o = -pg -fprofile-arcs -ftest-coverage CFLAGS_REMOVE_um_vdso.o = -pg -fprofile-arcs -ftest-coverage
targets += vdso-syms.lds
extra-$(VDSO64-y) += vdso-syms.lds
#
# Match symbols in the DSO that look like VDSO*; produce a file of constants.
#
sed-vdsosym := -e 's/^00*/0/' \
-e 's/^\([0-9a-fA-F]*\) . \(VDSO[a-zA-Z0-9_]*\)$$/\2 = 0x\1;/p'
quiet_cmd_vdsosym = VDSOSYM $@
define cmd_vdsosym
$(NM) $< | LC_ALL=C sed -n $(sed-vdsosym) | LC_ALL=C sort > $@
endef
$(obj)/%-syms.lds: $(obj)/%.so.dbg FORCE
$(call if_changed,vdsosym)
# #
# The DSO images are built using a special linker script. # The DSO images are built using a special linker script.
# #
......
...@@ -88,6 +88,7 @@ struct orc_entry { ...@@ -88,6 +88,7 @@ struct orc_entry {
unsigned sp_reg:4; unsigned sp_reg:4;
unsigned bp_reg:4; unsigned bp_reg:4;
unsigned type:2; unsigned type:2;
unsigned end:1;
} __packed; } __packed;
/* /*
...@@ -101,6 +102,7 @@ struct unwind_hint { ...@@ -101,6 +102,7 @@ struct unwind_hint {
s16 sp_offset; s16 sp_offset;
u8 sp_reg; u8 sp_reg;
u8 type; u8 type;
u8 end;
}; };
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -1157,6 +1157,7 @@ static int read_unwind_hints(struct objtool_file *file) ...@@ -1157,6 +1157,7 @@ static int read_unwind_hints(struct objtool_file *file)
cfa->offset = hint->sp_offset; cfa->offset = hint->sp_offset;
insn->state.type = hint->type; insn->state.type = hint->type;
insn->state.end = hint->end;
} }
return 0; return 0;
......
...@@ -31,7 +31,7 @@ struct insn_state { ...@@ -31,7 +31,7 @@ struct insn_state {
int stack_size; int stack_size;
unsigned char type; unsigned char type;
bool bp_scratch; bool bp_scratch;
bool drap; bool drap, end;
int drap_reg, drap_offset; int drap_reg, drap_offset;
struct cfi_reg vals[CFI_NUM_REGS]; struct cfi_reg vals[CFI_NUM_REGS];
}; };
......
...@@ -203,7 +203,8 @@ int orc_dump(const char *_objname) ...@@ -203,7 +203,8 @@ int orc_dump(const char *_objname)
print_reg(orc[i].bp_reg, orc[i].bp_offset); print_reg(orc[i].bp_reg, orc[i].bp_offset);
printf(" type:%s\n", orc_type_name(orc[i].type)); printf(" type:%s end:%d\n",
orc_type_name(orc[i].type), orc[i].end);
} }
elf_end(elf); elf_end(elf);
......
...@@ -31,6 +31,8 @@ int create_orc(struct objtool_file *file) ...@@ -31,6 +31,8 @@ int create_orc(struct objtool_file *file)
struct cfi_reg *cfa = &insn->state.cfa; struct cfi_reg *cfa = &insn->state.cfa;
struct cfi_reg *bp = &insn->state.regs[CFI_BP]; struct cfi_reg *bp = &insn->state.regs[CFI_BP];
orc->end = insn->state.end;
if (cfa->base == CFI_UNDEFINED) { if (cfa->base == CFI_UNDEFINED) {
orc->sp_reg = ORC_REG_UNDEFINED; orc->sp_reg = ORC_REG_UNDEFINED;
continue; continue;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment