Commit 08685be7 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc/64s: fix scv entry fallback flush vs interrupt

The L1D flush fallback functions are not recoverable vs interrupts,
yet the scv entry flush runs with MSR[EE]=1. This can result in a
timer (soft-NMI) or MCE or SRESET interrupt hitting here and overwriting
the EXRFI save area, which ends up corrupting userspace registers for
scv return.

Fix this by disabling RI and EE for the scv entry fallback flush.

Fixes: f7964378 ("powerpc/64s: flush L1D on kernel entry")
Cc: stable@vger.kernel.org # 5.9+ which also have flush L1D patch backport
Reported-by: default avatarTulio Magno Quites Machado Filho <tuliom@linux.ibm.com>
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210111062408.287092-1-npiggin@gmail.com
parent dd3a44c0
...@@ -63,6 +63,12 @@ ...@@ -63,6 +63,12 @@
nop; \ nop; \
nop; nop;
#define SCV_ENTRY_FLUSH_SLOT \
SCV_ENTRY_FLUSH_FIXUP_SECTION; \
nop; \
nop; \
nop;
/* /*
* r10 must be free to use, r13 must be paca * r10 must be free to use, r13 must be paca
*/ */
...@@ -70,6 +76,13 @@ ...@@ -70,6 +76,13 @@
STF_ENTRY_BARRIER_SLOT; \ STF_ENTRY_BARRIER_SLOT; \
ENTRY_FLUSH_SLOT ENTRY_FLUSH_SLOT
/*
* r10, ctr must be free to use, r13 must be paca
*/
#define SCV_INTERRUPT_TO_KERNEL \
STF_ENTRY_BARRIER_SLOT; \
SCV_ENTRY_FLUSH_SLOT
/* /*
* Macros for annotating the expected destination of (h)rfid * Macros for annotating the expected destination of (h)rfid
* *
......
...@@ -240,6 +240,14 @@ label##3: \ ...@@ -240,6 +240,14 @@ label##3: \
FTR_ENTRY_OFFSET 957b-958b; \ FTR_ENTRY_OFFSET 957b-958b; \
.popsection; .popsection;
#define SCV_ENTRY_FLUSH_FIXUP_SECTION \
957: \
.pushsection __scv_entry_flush_fixup,"a"; \
.align 2; \
958: \
FTR_ENTRY_OFFSET 957b-958b; \
.popsection;
#define RFI_FLUSH_FIXUP_SECTION \ #define RFI_FLUSH_FIXUP_SECTION \
951: \ 951: \
.pushsection __rfi_flush_fixup,"a"; \ .pushsection __rfi_flush_fixup,"a"; \
...@@ -273,10 +281,12 @@ label##3: \ ...@@ -273,10 +281,12 @@ label##3: \
extern long stf_barrier_fallback; extern long stf_barrier_fallback;
extern long entry_flush_fallback; extern long entry_flush_fallback;
extern long scv_entry_flush_fallback;
extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup; extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;
extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup; extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;
extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup; extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup;
extern long __start___entry_flush_fixup, __stop___entry_flush_fixup; extern long __start___entry_flush_fixup, __stop___entry_flush_fixup;
extern long __start___scv_entry_flush_fixup, __stop___scv_entry_flush_fixup;
extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup; extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
extern long __start__btb_flush_fixup, __stop__btb_flush_fixup; extern long __start__btb_flush_fixup, __stop__btb_flush_fixup;
......
...@@ -75,7 +75,7 @@ BEGIN_FTR_SECTION ...@@ -75,7 +75,7 @@ BEGIN_FTR_SECTION
bne .Ltabort_syscall bne .Ltabort_syscall
END_FTR_SECTION_IFSET(CPU_FTR_TM) END_FTR_SECTION_IFSET(CPU_FTR_TM)
#endif #endif
INTERRUPT_TO_KERNEL SCV_INTERRUPT_TO_KERNEL
mr r10,r1 mr r10,r1
ld r1,PACAKSAVE(r13) ld r1,PACAKSAVE(r13)
std r10,0(r1) std r10,0(r1)
......
...@@ -2993,6 +2993,25 @@ TRAMP_REAL_BEGIN(entry_flush_fallback) ...@@ -2993,6 +2993,25 @@ TRAMP_REAL_BEGIN(entry_flush_fallback)
ld r11,PACA_EXRFI+EX_R11(r13) ld r11,PACA_EXRFI+EX_R11(r13)
blr blr
/*
* The SCV entry flush happens with interrupts enabled, so it must disable
* to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10
* (containing LR) does not need to be preserved here because scv entry
* puts 0 in the pt_regs, CTR can be clobbered for the same reason.
*/
TRAMP_REAL_BEGIN(scv_entry_flush_fallback)
li r10,0
mtmsrd r10,1
lbz r10,PACAIRQHAPPENED(r13)
ori r10,r10,PACA_IRQ_HARD_DIS
stb r10,PACAIRQHAPPENED(r13)
std r11,PACA_EXRFI+EX_R11(r13)
L1D_DISPLACEMENT_FLUSH
ld r11,PACA_EXRFI+EX_R11(r13)
li r10,MSR_RI
mtmsrd r10,1
blr
TRAMP_REAL_BEGIN(rfi_flush_fallback) TRAMP_REAL_BEGIN(rfi_flush_fallback)
SET_SCRATCH0(r13); SET_SCRATCH0(r13);
GET_PACA(r13); GET_PACA(r13);
......
...@@ -145,6 +145,13 @@ SECTIONS ...@@ -145,6 +145,13 @@ SECTIONS
__stop___entry_flush_fixup = .; __stop___entry_flush_fixup = .;
} }
. = ALIGN(8);
__scv_entry_flush_fixup : AT(ADDR(__scv_entry_flush_fixup) - LOAD_OFFSET) {
__start___scv_entry_flush_fixup = .;
*(__scv_entry_flush_fixup)
__stop___scv_entry_flush_fixup = .;
}
. = ALIGN(8); . = ALIGN(8);
__stf_exit_barrier_fixup : AT(ADDR(__stf_exit_barrier_fixup) - LOAD_OFFSET) { __stf_exit_barrier_fixup : AT(ADDR(__stf_exit_barrier_fixup) - LOAD_OFFSET) {
__start___stf_exit_barrier_fixup = .; __start___stf_exit_barrier_fixup = .;
......
...@@ -290,9 +290,6 @@ void do_entry_flush_fixups(enum l1d_flush_type types) ...@@ -290,9 +290,6 @@ void do_entry_flush_fixups(enum l1d_flush_type types)
long *start, *end; long *start, *end;
int i; int i;
start = PTRRELOC(&__start___entry_flush_fixup);
end = PTRRELOC(&__stop___entry_flush_fixup);
instrs[0] = 0x60000000; /* nop */ instrs[0] = 0x60000000; /* nop */
instrs[1] = 0x60000000; /* nop */ instrs[1] = 0x60000000; /* nop */
instrs[2] = 0x60000000; /* nop */ instrs[2] = 0x60000000; /* nop */
...@@ -312,6 +309,8 @@ void do_entry_flush_fixups(enum l1d_flush_type types) ...@@ -312,6 +309,8 @@ void do_entry_flush_fixups(enum l1d_flush_type types)
if (types & L1D_FLUSH_MTTRIG) if (types & L1D_FLUSH_MTTRIG)
instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */
start = PTRRELOC(&__start___entry_flush_fixup);
end = PTRRELOC(&__stop___entry_flush_fixup);
for (i = 0; start < end; start++, i++) { for (i = 0; start < end; start++, i++) {
dest = (void *)start + *start; dest = (void *)start + *start;
...@@ -328,6 +327,25 @@ void do_entry_flush_fixups(enum l1d_flush_type types) ...@@ -328,6 +327,25 @@ void do_entry_flush_fixups(enum l1d_flush_type types)
patch_instruction((struct ppc_inst *)(dest + 2), ppc_inst(instrs[2])); patch_instruction((struct ppc_inst *)(dest + 2), ppc_inst(instrs[2]));
} }
start = PTRRELOC(&__start___scv_entry_flush_fixup);
end = PTRRELOC(&__stop___scv_entry_flush_fixup);
for (; start < end; start++, i++) {
dest = (void *)start + *start;
pr_devel("patching dest %lx\n", (unsigned long)dest);
patch_instruction((struct ppc_inst *)dest, ppc_inst(instrs[0]));
if (types == L1D_FLUSH_FALLBACK)
patch_branch((struct ppc_inst *)(dest + 1), (unsigned long)&scv_entry_flush_fallback,
BRANCH_SET_LINK);
else
patch_instruction((struct ppc_inst *)(dest + 1), ppc_inst(instrs[1]));
patch_instruction((struct ppc_inst *)(dest + 2), ppc_inst(instrs[2]));
}
printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i, printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i,
(types == L1D_FLUSH_NONE) ? "no" : (types == L1D_FLUSH_NONE) ? "no" :
(types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" :
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment